summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPrashanth Pai <ppai@redhat.com>2014-03-12 16:54:30 +0530
committerLuis Pabon <lpabon@redhat.com>2014-03-13 05:09:19 -0700
commit2ccd2c4d969cdd6e7feedd21ac2e5cb8498ff37d (patch)
tree1886732cf56e26ab836f9fd67c853eff2005fbeb
parent1a6b55714fddf7a1b526a31ee3f27589f01e98e5 (diff)
Sync with OpenStack Swift v1.13.0
Also, bumped version of gluster-swift to v1.13.0 Change-Id: I797dc704c9523540cba847b1e8ff3da97b79630c Signed-off-by: Prashanth Pai <ppai@redhat.com> Reviewed-on: http://review.gluster.org/7229 Reviewed-by: Chetan Risbud <crisbud@redhat.com> Reviewed-by: Luis Pabon <lpabon@redhat.com> Tested-by: Luis Pabon <lpabon@redhat.com>
-rw-r--r--doc/markdown/openstack_swift_sync.md9
-rw-r--r--gluster/swift/__init__.py2
-rw-r--r--test/functional/swift_test_client.py23
-rw-r--r--test/functional/swift_testing.py33
-rwxr-xr-xtest/functional/test_account.py148
-rw-r--r--test/functional/tests.py12
-rw-r--r--test/unit/proxy/controllers/test_account.py89
-rwxr-xr-xtest/unit/proxy/controllers/test_obj.py13
-rw-r--r--test/unit/proxy/test_server.py840
-rw-r--r--tools/requirements.txt1
-rw-r--r--tools/test-requires6
-rw-r--r--tox.ini2
12 files changed, 368 insertions, 810 deletions
diff --git a/doc/markdown/openstack_swift_sync.md b/doc/markdown/openstack_swift_sync.md
index f3feac8..ba2fadb 100644
--- a/doc/markdown/openstack_swift_sync.md
+++ b/doc/markdown/openstack_swift_sync.md
@@ -14,7 +14,8 @@ $ python setup.py sdist
$ ls dist
```
-* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net
+* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net.
+* Alternatively, if we are syncing with a Swift version which is already released, we can get the tar.gz file from Swift launchpad page and upload the same to gluster-swift launchpad.
## Setup Tox
Now that the swift source is availabe on launchpad.net, copy its link location and update tox.ini in gluster-swift with the new link.
@@ -22,7 +23,11 @@ Now that the swift source is availabe on launchpad.net, copy its link location a
## Update tests
This part is a little more complicated and now we need to *merge* the latest tests with ours.
-I suggest using a tool called `meld` to make this work easier.
+[meld](http://meldmerge.org/) is a great tool to make this work easier. The 3-way comparison feature of meld comes handy to compare 3 version of same file from:
+
+* Latest swift (say v1.13)
+* Previous swift (say v1.12)
+* gluster-swift (v1.12)
Files that need to be merged:
diff --git a/gluster/swift/__init__.py b/gluster/swift/__init__.py
index 4fe17ba..17f2fcf 100644
--- a/gluster/swift/__init__.py
+++ b/gluster/swift/__init__.py
@@ -45,6 +45,6 @@ class PkgInfo(object):
###
### Change the Package version here
###
-_pkginfo = PkgInfo('1.12.0', '0', 'gluster_swift', False)
+_pkginfo = PkgInfo('1.13.0', '0', 'gluster_swift', False)
__version__ = _pkginfo.pretty_version
__canonical_version__ = _pkginfo.canonical_version
diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py
index 47e023e..ecb2a6d 100644
--- a/test/functional/swift_test_client.py
+++ b/test/functional/swift_test_client.py
@@ -40,7 +40,7 @@ class RequestError(Exception):
class ResponseError(Exception):
- def __init__(self, response, method, path):
+ def __init__(self, response, method=None, path=None):
self.status = response.status
self.reason = response.reason
self.method = method
@@ -310,10 +310,11 @@ class Base:
def __str__(self):
return self.name
- def header_fields(self, fields):
+ def header_fields(self, required_fields, optional_fields=()):
headers = dict(self.conn.response.getheaders())
ret = {}
- for field in fields:
+
+ for field in required_fields:
if field[1] not in headers:
raise ValueError("%s was not found in response header" %
(field[1]))
@@ -322,6 +323,15 @@ class Base:
ret[field[0]] = int(headers[field[1]])
except ValueError:
ret[field[0]] = headers[field[1]]
+
+ for field in optional_fields:
+ if field[1] not in headers:
+ continue
+ try:
+ ret[field[0]] = int(headers[field[1]])
+ except ValueError:
+ ret[field[0]] = headers[field[1]]
+
return ret
@@ -480,10 +490,11 @@ class Container(Base):
parms=parms, cfg=cfg)
if self.conn.response.status == 204:
- fields = [['bytes_used', 'x-container-bytes-used'],
- ['object_count', 'x-container-object-count']]
+ required_fields = [['bytes_used', 'x-container-bytes-used'],
+ ['object_count', 'x-container-object-count']]
+ optional_fields = [['versions', 'x-versions-location']]
- return self.header_fields(fields)
+ return self.header_fields(required_fields, optional_fields)
raise ResponseError(self.conn.response, 'HEAD',
self.conn.make_path(self.path))
diff --git a/test/functional/swift_testing.py b/test/functional/swift_testing.py
index b5642b3..f05cb48 100644
--- a/test/functional/swift_testing.py
+++ b/test/functional/swift_testing.py
@@ -18,6 +18,7 @@ import os
import socket
import sys
from time import sleep
+from urlparse import urlparse
from test import get_config
@@ -119,18 +120,23 @@ conn = [None, None, None]
def retry(func, *args, **kwargs):
"""
- You can use the kwargs to override the 'retries' (default: 5) and
- 'use_account' (default: 1).
+ You can use the kwargs to override:
+ 'retries' (default: 5)
+ 'use_account' (default: 1) - which user's token to pass
+ 'url_account' (default: matches 'use_account') - which user's storage URL
+ 'resource' (default: url[url_account] - URL to connect to; retry()
+ will interpolate the variable :storage_url: if present
"""
global url, token, parsed, conn
retries = kwargs.get('retries', 5)
- use_account = 1
- if 'use_account' in kwargs:
- use_account = kwargs['use_account']
- del kwargs['use_account']
- use_account -= 1
- attempts = 0
- backoff = 1
+ attempts, backoff = 0, 1
+
+ # use account #1 by default; turn user's 1-indexed account into 0-indexed
+ use_account = kwargs.pop('use_account', 1) - 1
+
+ # access our own account by default
+ url_account = kwargs.pop('url_account', use_account + 1) - 1
+
while attempts <= retries:
attempts += 1
try:
@@ -146,8 +152,13 @@ def retry(func, *args, **kwargs):
if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \
http_connection(url[use_account])
- return func(url[use_account], token[use_account],
- parsed[use_account], conn[use_account],
+
+ # default resource is the account url[url_account]
+ resource = kwargs.pop('resource', '%(storage_url)s')
+ template_vars = {'storage_url': url[url_account]}
+ parsed_result = urlparse(resource % template_vars)
+ return func(url[url_account], token[use_account],
+ parsed_result, conn[url_account],
*args, **kwargs)
except (socket.error, HTTPException):
if attempts > retries:
diff --git a/test/functional/test_account.py b/test/functional/test_account.py
index b2f743f..d456090 100755
--- a/test/functional/test_account.py
+++ b/test/functional/test_account.py
@@ -16,12 +16,16 @@
# limitations under the License.
import unittest
+import json
from nose import SkipTest
from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
-
+from swift.common.middleware.acl import format_acl
+from test.functional.swift_test_client import Connection
+from test import get_config
from swift_testing import check_response, retry, skip, web_front_end
+import swift_testing
class TestAccount(unittest.TestCase):
@@ -66,6 +70,148 @@ class TestAccount(unittest.TestCase):
self.assert_(resp.status in (200, 204), resp.status)
self.assertEquals(resp.getheader('x-account-meta-test'), 'Value')
+ def test_tempauth_account_acls(self):
+ if skip:
+ raise SkipTest
+
+ # Determine whether this cluster has account ACLs; if not, skip test
+ conn = Connection(get_config('func_test'))
+ conn.authenticate()
+ status = conn.make_request(
+ 'GET', '/info', cfg={'verbatim_path': True})
+ if status // 100 != 2:
+ # Can't tell if account ACLs are enabled; skip tests proactively.
+ raise SkipTest
+ else:
+ cluster_info = json.loads(conn.response.read())
+ if not cluster_info.get('tempauth', {}).get('account_acls'):
+ raise SkipTest
+ if 'keystoneauth' in cluster_info:
+ # Unfortunate hack -- tempauth (with account ACLs) is expected
+ # to play nice with Keystone (without account ACLs), but Zuul
+ # functest framework doesn't give us an easy way to get a
+ # tempauth user.
+ raise SkipTest
+
+ def post(url, token, parsed, conn, headers):
+ new_headers = dict({'X-Auth-Token': token}, **headers)
+ conn.request('POST', parsed.path, '', new_headers)
+ return check_response(conn)
+
+ def put(url, token, parsed, conn, headers):
+ new_headers = dict({'X-Auth-Token': token}, **headers)
+ conn.request('PUT', parsed.path, '', new_headers)
+ return check_response(conn)
+
+ def delete(url, token, parsed, conn, headers):
+ new_headers = dict({'X-Auth-Token': token}, **headers)
+ conn.request('DELETE', parsed.path, '', new_headers)
+ return check_response(conn)
+
+ def head(url, token, parsed, conn):
+ conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
+ return check_response(conn)
+
+ def get(url, token, parsed, conn):
+ conn.request('GET', parsed.path, '', {'X-Auth-Token': token})
+ return check_response(conn)
+
+ try:
+ # User1 can POST to their own account (and reset the ACLs)
+ resp = retry(post, headers={'X-Account-Access-Control': '{}'},
+ use_account=1)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+ self.assertEqual(resp.getheader('X-Account-Access-Control'), None)
+
+ # User1 can GET their own empty account
+ resp = retry(get, use_account=1)
+ resp.read()
+ self.assertEqual(resp.status // 100, 2)
+ self.assertEqual(resp.getheader('X-Account-Access-Control'), None)
+
+ # User2 can't GET User1's account
+ resp = retry(get, use_account=2, url_account=1)
+ resp.read()
+ self.assertEqual(resp.status, 403)
+
+ # User1 is swift_owner of their own account, so they can POST an
+ # ACL -- let's do this and make User2 (test_user[1]) an admin
+ acl_user = swift_testing.swift_test_user[1]
+ acl = {'admin': [acl_user]}
+ headers = {'x-account-access-control': format_acl(
+ version=2, acl_dict=acl)}
+ resp = retry(post, headers=headers, use_account=1)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ # User1 can see the new header
+ resp = retry(get, use_account=1)
+ resp.read()
+ self.assertEqual(resp.status // 100, 2)
+ data_from_headers = resp.getheader('x-account-access-control')
+ expected = json.dumps(acl, separators=(',', ':'))
+ self.assertEqual(data_from_headers, expected)
+
+ # Now User2 should be able to GET the account and see the ACL
+ resp = retry(head, use_account=2, url_account=1)
+ resp.read()
+ data_from_headers = resp.getheader('x-account-access-control')
+ self.assertEqual(data_from_headers, expected)
+
+ # Revoke User2's admin access, grant User2 read-write access
+ acl = {'read-write': [acl_user]}
+ headers = {'x-account-access-control': format_acl(
+ version=2, acl_dict=acl)}
+ resp = retry(post, headers=headers, use_account=1)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ # User2 can still GET the account, but not see the ACL
+ # (since it's privileged data)
+ resp = retry(head, use_account=2, url_account=1)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+ self.assertEqual(resp.getheader('x-account-access-control'), None)
+
+ # User2 can PUT and DELETE a container
+ resp = retry(put, use_account=2, url_account=1,
+ resource='%(storage_url)s/mycontainer', headers={})
+ resp.read()
+ self.assertEqual(resp.status, 201)
+ resp = retry(delete, use_account=2, url_account=1,
+ resource='%(storage_url)s/mycontainer', headers={})
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ # Revoke User2's read-write access, grant User2 read-only access
+ acl = {'read-only': [acl_user]}
+ headers = {'x-account-access-control': format_acl(
+ version=2, acl_dict=acl)}
+ resp = retry(post, headers=headers, use_account=1)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ # User2 can still GET the account, but not see the ACL
+ # (since it's privileged data)
+ resp = retry(head, use_account=2, url_account=1)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+ self.assertEqual(resp.getheader('x-account-access-control'), None)
+
+ # User2 can't PUT a container
+ resp = retry(put, use_account=2, url_account=1,
+ resource='%(storage_url)s/mycontainer', headers={})
+ resp.read()
+ self.assertEqual(resp.status, 403)
+
+ finally:
+ # Make sure to clean up even if tests fail -- User2 should not
+ # have access to User1's account in other functional tests!
+ resp = retry(post, headers={'X-Account-Access-Control': '{}'},
+ use_account=1)
+ resp.read()
+
def test_unicode_metadata(self):
if skip:
raise SkipTest
diff --git a/test/functional/tests.py b/test/functional/tests.py
index 9469cff..0d9a9ef 100644
--- a/test/functional/tests.py
+++ b/test/functional/tests.py
@@ -149,7 +149,7 @@ def timeout(seconds, method, *args, **kwargs):
return False
-class Utils:
+class Utils(object):
@classmethod
def create_ascii_name(cls, length=None):
return uuid.uuid4().hex
@@ -201,7 +201,7 @@ class Base2(object):
Utils.create_name = Utils.create_ascii_name
-class TestAccountEnv:
+class TestAccountEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(config)
@@ -388,7 +388,7 @@ class TestAccountUTF8(Base2, TestAccount):
set_up = False
-class TestAccountNoContainersEnv:
+class TestAccountNoContainersEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(config)
@@ -417,7 +417,7 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):
set_up = False
-class TestContainerEnv:
+class TestContainerEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(config)
@@ -708,7 +708,7 @@ class TestContainerUTF8(Base2, TestContainer):
set_up = False
-class TestContainerPathsEnv:
+class TestContainerPathsEnv(object):
@classmethod
def setUp(cls):
raise SkipTest('Objects ending in / are not supported')
@@ -888,7 +888,7 @@ class TestContainerPaths(Base):
['dir1/subdir with spaces/file B'])
-class TestFileEnv:
+class TestFileEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(config)
diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py
index eefd57d..47f76dc 100644
--- a/test/unit/proxy/controllers/test_account.py
+++ b/test/unit/proxy/controllers/test_account.py
@@ -16,12 +16,14 @@
import mock
import unittest
-from swift.common.swob import Request
+from swift.common.swob import Request, Response
+from swift.common.middleware.acl import format_acl
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info
from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN
from test.unit import fake_http_connect, FakeRing, FakeMemcache
from swift.common.request_helpers import get_sys_meta_prefix
+import swift.proxy.controllers.base
class TestAccountController(unittest.TestCase):
@@ -152,6 +154,91 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+ def _make_user_and_sys_acl_headers_data(self):
+ acl = {
+ 'admin': ['AUTH_alice', 'AUTH_bob'],
+ 'read-write': ['AUTH_carol'],
+ 'read-only': [],
+ }
+ user_prefix = 'x-account-' # external, user-facing
+ user_headers = {(user_prefix + 'access-control'): format_acl(
+ version=2, acl_dict=acl)}
+ sys_prefix = get_sys_meta_prefix('account') # internal, system-facing
+ sys_headers = {(sys_prefix + 'core-access-control'): format_acl(
+ version=2, acl_dict=acl)}
+ return user_headers, sys_headers
+
+ def test_account_acl_headers_translated_for_GET_HEAD(self):
+ # Verify that a GET/HEAD which receives X-Account-Sysmeta-Acl-* headers
+ # from the account server will remap those headers to X-Account-Acl-*
+
+ hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data()
+ controller = proxy_server.AccountController(self.app, 'acct')
+
+ for verb in ('GET', 'HEAD'):
+ req = Request.blank('/v1/acct', environ={'swift_owner': True})
+ controller.GETorHEAD_base = lambda *_: Response(
+ headers=hdrs_int, environ={
+ 'PATH_INFO': '/acct',
+ 'REQUEST_METHOD': verb,
+ })
+ method = getattr(controller, verb)
+ resp = method(req)
+ for header, value in hdrs_ext.items():
+ if value:
+ self.assertEqual(resp.headers.get(header), value)
+ else:
+ # blank ACLs should result in no header
+ self.assert_(header not in resp.headers)
+
+ def test_add_acls_impossible_cases(self):
+ # For test coverage: verify that defensive coding does defend, in cases
+ # that shouldn't arise naturally
+
+ # add_acls should do nothing if REQUEST_METHOD isn't HEAD/GET/PUT/POST
+ resp = Response()
+ controller = proxy_server.AccountController(self.app, 'a')
+ resp.environ['PATH_INFO'] = '/a'
+ resp.environ['REQUEST_METHOD'] = 'OPTIONS'
+ controller.add_acls_from_sys_metadata(resp)
+ self.assertEqual(1, len(resp.headers)) # we always get Content-Type
+ self.assertEqual(2, len(resp.environ))
+
+ def test_memcache_key_impossible_cases(self):
+ # For test coverage: verify that defensive coding does defend, in cases
+ # that shouldn't arise naturally
+ self.assertRaises(
+ ValueError,
+ lambda: swift.proxy.controllers.base.get_container_memcache_key(
+ '/a', None))
+
+ def test_stripping_swift_admin_headers(self):
+ # Verify that a GET/HEAD which receives privileged headers from the
+ # account server will strip those headers for non-swift_owners
+
+ hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data()
+ headers = {
+ 'x-account-meta-harmless': 'hi mom',
+ 'x-account-meta-temp-url-key': 's3kr1t',
+ }
+ controller = proxy_server.AccountController(self.app, 'acct')
+
+ for verb in ('GET', 'HEAD'):
+ for env in ({'swift_owner': True}, {'swift_owner': False}):
+ req = Request.blank('/v1/acct', environ=env)
+ controller.GETorHEAD_base = lambda *_: Response(
+ headers=headers, environ={
+ 'PATH_INFO': '/acct',
+ 'REQUEST_METHOD': verb,
+ })
+ method = getattr(controller, verb)
+ resp = method(req)
+ self.assertEqual(resp.headers.get('x-account-meta-harmless'),
+ 'hi mom')
+ privileged_header_present = (
+ 'x-account-meta-temp-url-key' in resp.headers)
+ self.assertEqual(privileged_header_present, env['swift_owner'])
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py
index cae62b0..aada616 100755
--- a/test/unit/proxy/controllers/test_obj.py
+++ b/test/unit/proxy/controllers/test_obj.py
@@ -21,6 +21,7 @@ import mock
import swift
from swift.proxy import server as proxy_server
+from swift.common.swob import HTTPException
from test.unit import FakeRing, FakeMemcache, fake_http_connect
@@ -44,7 +45,7 @@ class TestObjControllerWriteAffinity(unittest.TestCase):
self.app = proxy_server.Application(
None, FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing(max_more_nodes=9))
- self.app.request_node_count = lambda ring: 10000000
+ self.app.request_node_count = lambda replicas: 10000000
self.app.sort_nodes = lambda l: l # stop shuffling the primary nodes
def test_iter_nodes_local_first_noops_when_no_affinity(self):
@@ -107,14 +108,20 @@ class TestObjController(unittest.TestCase):
# and now test that we add the header to log_info
req = swift.common.swob.Request.blank('/v1/a/c/o')
req.headers['x-copy-from'] = 'somewhere'
- controller.PUT(req)
+ try:
+ controller.PUT(req)
+ except HTTPException:
+ pass
self.assertEquals(
req.environ.get('swift.log_info'), ['x-copy-from:somewhere'])
# and then check that we don't do that for originating POSTs
req = swift.common.swob.Request.blank('/v1/a/c/o')
req.method = 'POST'
req.headers['x-copy-from'] = 'elsewhere'
- controller.PUT(req)
+ try:
+ controller.PUT(req)
+ except HTTPException:
+ pass
self.assertEquals(req.environ.get('swift.log_info'), None)
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index b68be9f..4086a32 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -46,7 +46,8 @@ from gluster.swift.container import server as container_server
from gluster.swift.obj import server as object_server
from swift.common import ring
from swift.common.middleware import proxy_logging
-from swift.common.exceptions import ChunkReadTimeout, SegmentError
+from swift.common.middleware.acl import parse_acl, format_acl
+from swift.common.exceptions import ChunkReadTimeout
from swift.common.constraints import MAX_META_NAME_LENGTH, \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \
@@ -54,38 +55,28 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \
from swift.common import utils
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import monkey_patch_mimetools
-from swift.proxy.controllers.obj import SegmentedIterable
from swift.proxy.controllers import base as proxy_base
from swift.proxy.controllers.base import get_container_memcache_key, \
get_account_memcache_key, cors_validation
import swift.proxy.controllers
-from swift.common.swob import Request, Response, HTTPNotFound, \
- HTTPUnauthorized
+from swift.common.request_helpers import get_sys_meta_prefix
+from swift.common.swob import Request, Response, HTTPUnauthorized
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
STATIC_TIME = time.time()
-_request_instances = weakref.WeakKeyDictionary()
_test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \
_testdir = _orig_SysLogHandler = None
-def request_init(self, *args, **kwargs):
- self._orig_init(*args, **kwargs)
-
- _request_instances[self] = None
-
-
def do_setup(the_object_server):
utils.HASH_PATH_SUFFIX = 'endcap'
global _testdir, _test_servers, _test_sockets, \
_orig_container_listing_limit, _test_coros, _orig_SysLogHandler
_orig_SysLogHandler = utils.SysLogHandler
utils.SysLogHandler = mock.MagicMock()
- Request._orig_init = Request.__init__
- Request.__init__ = request_init
monkey_patch_mimetools()
# Since we're starting up a lot here, we're going to test more than
# just chunked puts; we're also going to test parts of
@@ -99,8 +90,6 @@ def do_setup(the_object_server):
mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
mkdirs(os.path.join(_testdir, 'a'))
mkdirs(os.path.join(_testdir, 'a', 'tmp'))
- _orig_container_listing_limit = \
- swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT
conf = {'devices': _testdir, 'swift_dir': _testdir,
'mount_check': 'false', 'allowed_headers':
'content-encoding, x-object-manifest, content-disposition, foo',
@@ -211,10 +200,7 @@ def setup():
def teardown():
for server in _test_coros:
server.kill()
- swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = \
- _orig_container_listing_limit
rmtree(os.path.dirname(_testdir))
- Request.__init__ = Request._orig_init
utils.SysLogHandler = _orig_SysLogHandler
@@ -1716,7 +1702,7 @@ class TestObjectController(unittest.TestCase):
except ChunkReadTimeout:
got_exc = True
self.assert_(not got_exc)
- self.app.node_timeout = 0.1
+ self.app.recoverable_node_timeout = 0.1
set_http_connect(200, 200, 200, slow=True)
resp = req.get_response(self.app)
got_exc = False
@@ -1731,7 +1717,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.app.update_request(req)
- self.app.node_timeout = 0.1
+ self.app.recoverable_node_timeout = 0.1
set_http_connect(200, 200, 200, slow=[3])
resp = req.get_response(self.app)
got_exc = False
@@ -3316,402 +3302,6 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
self.assertEquals(headers[:len(exp)], exp)
- def test_chunked_put_lobjects_with_nonzero_size_manifest_file(self):
- raise SkipTest("Not until we support pure object requests")
- # Create a container for our segmented/manifest object testing
- (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \
- _test_sockets
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented_nonzero HTTP/1.1\r\nHost: localhost\r\n'
- 'Connection: close\r\nX-Storage-Token: t\r\n'
- 'Content-Length: 0\r\n\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- # Create the object segments
- segment_etags = []
- for segment in xrange(5):
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented_nonzero/name/%s HTTP/1.1\r\nHost: '
- 'localhost\r\nConnection: close\r\nX-Storage-Token: '
- 't\r\nContent-Length: 5\r\n\r\n1234 ' % str(segment))
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- segment_etags.append(md5('1234 ').hexdigest())
-
- # Create the nonzero size manifest file
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
- 'localhost\r\nConnection: close\r\nX-Storage-Token: '
- 't\r\nContent-Length: 5\r\n\r\nabcd ')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
-
- # Create the object manifest file
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('POST /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
- 'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n'
- 'X-Object-Manifest: segmented_nonzero/name/\r\n'
- 'Foo: barbaz\r\nContent-Type: text/jibberish\r\n'
- '\r\n\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 202'
- self.assertEquals(headers[:len(exp)], exp)
-
- # Ensure retrieving the manifest file gets the whole object
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
- 'localhost\r\nConnection: close\r\nX-Auth-Token: '
- 't\r\n\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
- self.assert_('Content-Type: text/jibberish' in headers)
- self.assert_('Foo: barbaz' in headers)
- expected_etag = md5(''.join(segment_etags)).hexdigest()
- self.assert_('Etag: "%s"' % expected_etag in headers)
- body = fd.read()
- self.assertEquals(body, '1234 1234 1234 1234 1234 ')
-
- # Get lobjects with Range smaller than manifest file
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
- 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
- 'Range: bytes=0-4\r\n\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 206'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
- self.assert_('Content-Type: text/jibberish' in headers)
- self.assert_('Foo: barbaz' in headers)
- expected_etag = md5(''.join(segment_etags)).hexdigest()
- body = fd.read()
- self.assertEquals(body, '1234 ')
-
- # Get lobjects with Range bigger than manifest file
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
- 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
- 'Range: bytes=11-15\r\n\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 206'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
- self.assert_('Content-Type: text/jibberish' in headers)
- self.assert_('Foo: barbaz' in headers)
- expected_etag = md5(''.join(segment_etags)).hexdigest()
- body = fd.read()
- self.assertEquals(body, '234 1')
-
- def test_chunked_put_lobjects(self):
- raise SkipTest("Not until we support pure object requests")
- # Create a container for our segmented/manifest object testing
- (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
- obj2lis) = _test_sockets
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented%20object HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: 0\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- # Create the object segments
- segment_etags = []
- for segment in xrange(5):
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented%%20object/object%%20name/%s '
- 'HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: 5\r\n'
- '\r\n'
- '1234 ' % str(segment))
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- segment_etags.append(md5('1234 ').hexdigest())
- # Create the object manifest file
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: 0\r\n'
- 'X-Object-Manifest: segmented%20object/object%20name/\r\n'
- 'Content-Type: text/jibberish\r\n'
- 'Foo: barbaz\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- # Check retrieving the listing the manifest would retrieve
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/segmented%20object?prefix=object%20name/ '
- 'HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- body = fd.read()
- self.assertEquals(
- body,
- 'object name/0\n'
- 'object name/1\n'
- 'object name/2\n'
- 'object name/3\n'
- 'object name/4\n')
- # Ensure retrieving the manifest file gets the whole object
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in
- headers)
- self.assert_('Content-Type: text/jibberish' in headers)
- self.assert_('Foo: barbaz' in headers)
- expected_etag = md5(''.join(segment_etags)).hexdigest()
- self.assert_('Etag: "%s"' % expected_etag in headers)
- body = fd.read()
- self.assertEquals(body, '1234 1234 1234 1234 1234 ')
- # Do it again but exceeding the container listing limit
- swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
-
- fd = sock.makefile()
- fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in
- headers)
- self.assert_('Content-Type: text/jibberish' in headers)
- body = fd.read()
- # A bit fragile of a test; as it makes the assumption that all
- # will be sent in a single chunk.
- self.assertEquals(
- body, '19\r\n1234 1234 1234 1234 1234 \r\n0\r\n\r\n')
- # Make a copy of the manifested object, which should
- # error since the number of segments exceeds
- # CONTAINER_LISTING_LIMIT.
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- 'X-Copy-From: segmented%20object/object%20name\r\n'
- 'Content-Length: 0\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 413'
- self.assertEquals(headers[:len(exp)], exp)
- body = fd.read()
- # After adjusting the CONTAINER_LISTING_LIMIT, make a copy of
- # the manifested object which should consolidate the segments.
- swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 10000
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- 'X-Copy-From: segmented%20object/object%20name\r\n'
- 'Content-Length: 0\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- body = fd.read()
- # Retrieve and validate the copy.
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/segmented%20object/copy HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('x-object-manifest:' not in headers.lower())
- self.assert_('Content-Length: 25\r' in headers)
- body = fd.read()
- self.assertEquals(body, '1234 1234 1234 1234 1234 ')
- # Create an object manifest file pointing to nothing
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/segmented%20object/empty HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: 0\r\n'
- 'X-Object-Manifest: segmented%20object/empty/\r\n'
- 'Content-Type: text/jibberish\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- # Ensure retrieving the manifest file gives a zero-byte file
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/segmented%20object/empty HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('X-Object-Manifest: segmented%20object/empty/' in headers)
- self.assert_('Content-Type: text/jibberish' in headers)
- body = fd.read()
- self.assertEquals(body, '')
- # Check copy content type
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/c/obj HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: 0\r\n'
- 'Content-Type: text/jibberish\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/c/obj2 HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: 0\r\n'
- 'X-Copy-From: c/obj\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- # Ensure getting the copied file gets original content-type
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/c/obj2 HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('Content-Type: text/jibberish' in headers)
- # Check set content type
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/c/obj3 HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: 0\r\n'
- 'Content-Type: foo/bar\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- # Ensure getting the copied file gets original content-type
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/c/obj3 HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('Content-Type: foo/bar' in
- headers.split('\r\n'), repr(headers.split('\r\n')))
- # Check set content type with charset
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/c/obj4 HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: 0\r\n'
- 'Content-Type: foo/bar; charset=UTF-8\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEquals(headers[:len(exp)], exp)
- # Ensure getting the copied file gets original content-type
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/c/obj4 HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEquals(headers[:len(exp)], exp)
- self.assert_('Content-Type: foo/bar; charset=UTF-8' in
- headers.split('\r\n'), repr(headers.split('\r\n')))
-
def test_mismatched_etags(self):
with save_globals():
# no etag supplied, object servers return success w/ diff values
@@ -4085,53 +3675,63 @@ class TestObjectController(unittest.TestCase):
self.assertTrue('X-Delete-At in past' in resp.body)
def test_leak_1(self):
- prolis = _test_sockets[0]
- prosrv = _test_servers[0]
- obj_len = prosrv.client_chunk_size * 2
- # PUT test file
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- 'Content-Length: %s\r\n'
- 'Content-Type: application/octet-stream\r\n'
- '\r\n%s' % (obj_len, 'a' * obj_len))
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEqual(headers[:len(exp)], exp)
- # Remember Request instance count, make sure the GC is run for pythons
- # without reference counting.
- for i in xrange(4):
- sleep(0) # let eventlet do its thing
- gc.collect()
- else:
- sleep(0)
- before_request_instances = len(_request_instances)
- # GET test file, but disconnect early
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Auth-Token: t\r\n'
- '\r\n')
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 200'
- self.assertEqual(headers[:len(exp)], exp)
- fd.read(1)
- fd.close()
- sock.close()
- # Make sure the GC is run again for pythons without reference counting
- for i in xrange(4):
- sleep(0) # let eventlet do its thing
- gc.collect()
- else:
- sleep(0)
- self.assertEquals(before_request_instances, len(_request_instances))
+ _request_instances = weakref.WeakKeyDictionary()
+ _orig_init = Request.__init__
+
+ def request_init(self, *args, **kwargs):
+ _orig_init(self, *args, **kwargs)
+ _request_instances[self] = None
+
+ with mock.patch.object(Request, "__init__", request_init):
+ prolis = _test_sockets[0]
+ prosrv = _test_servers[0]
+ obj_len = prosrv.client_chunk_size * 2
+ # PUT test file
+ sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+ fd = sock.makefile()
+ fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n'
+ 'Host: localhost\r\n'
+ 'Connection: close\r\n'
+ 'X-Auth-Token: t\r\n'
+ 'Content-Length: %s\r\n'
+ 'Content-Type: application/octet-stream\r\n'
+ '\r\n%s' % (obj_len, 'a' * obj_len))
+ fd.flush()
+ headers = readuntil2crlfs(fd)
+ exp = 'HTTP/1.1 201'
+ self.assertEqual(headers[:len(exp)], exp)
+ # Remember Request instance count, make sure the GC is run for
+ # pythons without reference counting.
+ for i in xrange(4):
+ sleep(0) # let eventlet do its thing
+ gc.collect()
+ else:
+ sleep(0)
+ before_request_instances = len(_request_instances)
+ # GET test file, but disconnect early
+ sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+ fd = sock.makefile()
+ fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n'
+ 'Host: localhost\r\n'
+ 'Connection: close\r\n'
+ 'X-Auth-Token: t\r\n'
+ '\r\n')
+ fd.flush()
+ headers = readuntil2crlfs(fd)
+ exp = 'HTTP/1.1 200'
+ self.assertEqual(headers[:len(exp)], exp)
+ fd.read(1)
+ fd.close()
+ sock.close()
+ # Make sure the GC is run again for pythons without reference
+ # counting
+ for i in xrange(4):
+ sleep(0) # let eventlet do its thing
+ gc.collect()
+ else:
+ sleep(0)
+ self.assertEquals(
+ before_request_instances, len(_request_instances))
def test_OPTIONS(self):
with save_globals():
@@ -4926,9 +4526,10 @@ class TestContainerController(unittest.TestCase):
controller = \
proxy_server.ContainerController(self.app, 'a', 'c')
set_http_connect(200, 201, 201, 201, give_connect=test_connect)
- req = Request.blank('/v1/a/c',
- environ={'REQUEST_METHOD': method},
- headers={test_header: test_value})
+ req = Request.blank(
+ '/v1/a/c',
+ environ={'REQUEST_METHOD': method, 'swift_owner': True},
+ headers={test_header: test_value})
self.app.update_request(req)
getattr(controller, method)(req)
self.assertEquals(test_errors, [])
@@ -6006,313 +5607,6 @@ class Stub(object):
pass
-class TestSegmentedIterable(unittest.TestCase):
-
- def setUp(self):
- self.controller = FakeObjectController()
-
- def test_load_next_segment_unexpected_error(self):
- # Iterator value isn't a dict
- self.assertRaises(Exception,
- SegmentedIterable(self.controller, None,
- [None])._load_next_segment)
- self.assert_(self.controller.exception_args[0].startswith(
- 'ERROR: While processing manifest'))
-
- def test_load_next_segment_with_no_segments(self):
- self.assertRaises(StopIteration,
- SegmentedIterable(self.controller, 'lc',
- [])._load_next_segment)
-
- def test_load_next_segment_with_one_segment(self):
- segit = SegmentedIterable(self.controller, 'lc', [{'name':
- 'o1'}])
- segit._load_next_segment()
- self.assertEquals(
- self.controller.GETorHEAD_base_args[0][4], '/a/lc/o1')
- data = ''.join(segit.segment_iter)
- self.assertEquals(data, '1')
-
- def test_load_next_segment_with_two_segments(self):
- segit = SegmentedIterable(self.controller, 'lc', [{'name':
- 'o1'}, {'name': 'o2'}])
- segit._load_next_segment()
- self.assertEquals(
- self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o1')
- data = ''.join(segit.segment_iter)
- self.assertEquals(data, '1')
- segit._load_next_segment()
- self.assertEquals(
- self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
- data = ''.join(segit.segment_iter)
- self.assertEquals(data, '22')
-
- def test_load_next_segment_rate_limiting(self):
- sleep_calls = []
-
- def _stub_sleep(sleepy_time):
- sleep_calls.append(sleepy_time)
- orig_sleep = swift.proxy.controllers.obj.sleep
- try:
- swift.proxy.controllers.obj.sleep = _stub_sleep
- segit = SegmentedIterable(
- self.controller, 'lc', [
- {'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'},
- {'name': 'o4'}, {'name': 'o5'}])
-
- # rate_limit_after_segment == 3, so the first 3 segments should
- # invoke no sleeping.
- for _ in xrange(3):
- segit._load_next_segment()
- self.assertEquals([], sleep_calls)
- self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
- '/a/lc/o3')
-
- # Loading of next (4th) segment starts rate-limiting.
- segit._load_next_segment()
- self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
- self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
- '/a/lc/o4')
-
- sleep_calls = []
- segit._load_next_segment()
- self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
- self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
- '/a/lc/o5')
- finally:
- swift.proxy.controllers.obj.sleep = orig_sleep
-
- def test_load_next_segment_range_req_rate_limiting(self):
- sleep_calls = []
-
- def _stub_sleep(sleepy_time):
- sleep_calls.append(sleepy_time)
- orig_sleep = swift.proxy.controllers.obj.sleep
- try:
- swift.proxy.controllers.obj.sleep = _stub_sleep
- segit = SegmentedIterable(
- self.controller, 'lc', [
- {'name': 'o0', 'bytes': 5}, {'name': 'o1', 'bytes': 5},
- {'name': 'o2', 'bytes': 1}, {'name': 'o3'}, {'name': 'o4'},
- {'name': 'o5'}, {'name': 'o6'}])
-
- # this tests for a range request which skips over the whole first
- # segment, after that 3 segments will be read in because the
- # rate_limit_after_segment == 3, then sleeping starts
- segit_iter = segit.app_iter_range(10, None)
- segit_iter.next()
- for _ in xrange(2):
- # this is set to 2 instead of 3 because o2 was loaded after
- # o0 and o1 were skipped.
- segit._load_next_segment()
- self.assertEquals([], sleep_calls)
- self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
- '/a/lc/o4')
-
- # Loading of next (5th) segment starts rate-limiting.
- segit._load_next_segment()
- self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
- self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
- '/a/lc/o5')
-
- sleep_calls = []
- segit._load_next_segment()
- self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
- self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
- '/a/lc/o6')
- finally:
- swift.proxy.controllers.obj.sleep = orig_sleep
-
- def test_load_next_segment_with_two_segments_skip_first(self):
- segit = SegmentedIterable(self.controller, 'lc', [{'name':
- 'o1'}, {'name': 'o2'}])
- segit.ratelimit_index = 0
- segit.listing.next()
- segit._load_next_segment()
- self.assertEquals(
- self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
- data = ''.join(segit.segment_iter)
- self.assertEquals(data, '22')
-
- def test_load_next_segment_with_seek(self):
- segit = SegmentedIterable(self.controller, 'lc',
- [{'name': 'o1', 'bytes': 1},
- {'name': 'o2', 'bytes': 2}])
- segit.ratelimit_index = 0
- segit.listing.next()
- segit.seek = 1
- segit._load_next_segment()
- self.assertEquals(
- self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
- self.assertEquals(
- str(self.controller.GETorHEAD_base_args[-1][0].range),
- 'bytes=1-')
- data = ''.join(segit.segment_iter)
- self.assertEquals(data, '2')
-
- def test_fetching_only_what_you_need(self):
- segit = SegmentedIterable(self.controller, 'lc',
- [{'name': 'o7', 'bytes': 7},
- {'name': 'o8', 'bytes': 8},
- {'name': 'o9', 'bytes': 9}])
-
- body = ''.join(segit.app_iter_range(10, 20))
- self.assertEqual('8888899999', body)
-
- GoH_args = self.controller.GETorHEAD_base_args
- self.assertEquals(2, len(GoH_args))
-
- # Either one is fine, as they both indicate "from byte 3 to (the last)
- # byte 8".
- self.assert_(str(GoH_args[0][0].range) in ['bytes=3-', 'bytes=3-8'])
-
- # This one must ask only for the bytes it needs; otherwise we waste
- # bandwidth pulling bytes from the object server and then throwing
- # them out
- self.assertEquals(str(GoH_args[1][0].range), 'bytes=0-4')
-
- def test_load_next_segment_with_get_error(self):
-
- def local_GETorHEAD_base(*args):
- return HTTPNotFound()
-
- self.controller.GETorHEAD_base = local_GETorHEAD_base
- self.assertRaises(Exception,
- SegmentedIterable(self.controller, 'lc',
- [{'name': 'o1'}])._load_next_segment)
- self.assert_(self.controller.exception_args[0].startswith(
- 'ERROR: While processing manifest'))
- self.assertEquals(str(self.controller.exception_info[1]),
- 'Could not load object segment /a/lc/o1: 404')
-
- def test_iter_unexpected_error(self):
- # Iterator value isn't a dict
- self.assertRaises(Exception, ''.join,
- SegmentedIterable(self.controller, None, [None]))
- self.assert_(self.controller.exception_args[0].startswith(
- 'ERROR: While processing manifest'))
-
- def test_iter_with_no_segments(self):
- segit = SegmentedIterable(self.controller, 'lc', [])
- self.assertEquals(''.join(segit), '')
-
- def test_iter_with_one_segment(self):
- segit = SegmentedIterable(self.controller, 'lc', [{'name':
- 'o1'}])
- segit.response = Stub()
- self.assertEquals(''.join(segit), '1')
-
- def test_iter_with_two_segments(self):
- segit = SegmentedIterable(self.controller, 'lc', [{'name':
- 'o1'}, {'name': 'o2'}])
- segit.response = Stub()
- self.assertEquals(''.join(segit), '122')
-
- def test_iter_with_get_error(self):
-
- def local_GETorHEAD_base(*args):
- return HTTPNotFound()
-
- self.controller.GETorHEAD_base = local_GETorHEAD_base
- self.assertRaises(Exception, ''.join,
- SegmentedIterable(self.controller, 'lc', [{'name':
- 'o1'}]))
- self.assert_(self.controller.exception_args[0].startswith(
- 'ERROR: While processing manifest'))
- self.assertEquals(str(self.controller.exception_info[1]),
- 'Could not load object segment /a/lc/o1: 404')
-
- def test_app_iter_range_unexpected_error(self):
- # Iterator value isn't a dict
- self.assertRaises(Exception,
- SegmentedIterable(self.controller, None,
- [None]).app_iter_range(None,
- None).next)
- self.assert_(self.controller.exception_args[0].startswith(
- 'ERROR: While processing manifest'))
-
- def test_app_iter_range_with_no_segments(self):
- self.assertEquals(''.join(SegmentedIterable(
- self.controller, 'lc', []).app_iter_range(None, None)), '')
- self.assertEquals(''.join(SegmentedIterable(
- self.controller, 'lc', []).app_iter_range(3, None)), '')
- self.assertEquals(''.join(SegmentedIterable(
- self.controller, 'lc', []).app_iter_range(3, 5)), '')
- self.assertEquals(''.join(SegmentedIterable(
- self.controller, 'lc', []).app_iter_range(None, 5)), '')
-
- def test_app_iter_range_with_one_segment(self):
- listing = [{'name': 'o1', 'bytes': 1}]
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(None, None)), '1')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- self.assertEquals(''.join(segit.app_iter_range(3, None)), '')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- self.assertEquals(''.join(segit.app_iter_range(3, 5)), '')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(None, 5)), '1')
-
- def test_app_iter_range_with_two_segments(self):
- listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}]
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(None, None)), '122')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(1, None)), '22')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(1, 5)), '22')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(None, 2)), '12')
-
- def test_app_iter_range_with_many_segments(self):
- listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2},
- {'name': 'o3', 'bytes': 3}, {'name': 'o4', 'bytes': 4},
- {'name': 'o5', 'bytes': 5}]
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(None, None)),
- '122333444455555')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(3, None)),
- '333444455555')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(5, None)), '3444455555')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(None, 6)), '122333')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(None, 7)), '1223334')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(3, 7)), '3334')
-
- segit = SegmentedIterable(self.controller, 'lc', listing)
- segit.response = Stub()
- self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34')
-
-
class TestProxyObjectPerformance(unittest.TestCase):
def setUp(self):
diff --git a/tools/requirements.txt b/tools/requirements.txt
index 1139866..bbac51a 100644
--- a/tools/requirements.txt
+++ b/tools/requirements.txt
@@ -5,4 +5,3 @@ netifaces>=0.5
pastedeploy>=1.3.3
simplejson>=2.0.9
xattr>=0.4
-python-swiftclient
diff --git a/tools/test-requires b/tools/test-requires
index d2c027f..63d499e 100644
--- a/tools/test-requires
+++ b/tools/test-requires
@@ -1,7 +1,4 @@
-# Install bounded pep8/pyflakes first, then let flake8 install
-pep8==1.4.5
-pyflakes==0.7.2
-flake8==2.0
+# Hacking already pins down pep8, pyflakes and flake8
hacking>=0.5.6,<0.6
coverage
nose
@@ -10,5 +7,6 @@ openstack.nose_plugin
nosehtmloutput
sphinx>=1.1.2,<1.2
mock>=0.8.0
+python-swiftclient
python-keystoneclient
prettytable
diff --git a/tox.ini b/tox.ini
index 81621a0..ebcb5a1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ setenv = VIRTUAL_ENV={envdir}
NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1
deps =
- https://launchpad.net/gluster-swift/icehouse/1.12.0/+download/swift-1.12.0.tar.gz
+ https://launchpad.net/gluster-swift/icehouse/1.13.0/+download/swift-1.13.0.tar.gz
--download-cache={homedir}/.pipcache
-r{toxinidir}/tools/test-requires
-r{toxinidir}/tools/requirements.txt