summaryrefslogtreecommitdiffstats
path: root/test/functional/__init__.py
diff options
context:
space:
mode:
Diffstat (limited to 'test/functional/__init__.py')
-rw-r--r--test/functional/__init__.py286
1 files changed, 234 insertions, 52 deletions
diff --git a/test/functional/__init__.py b/test/functional/__init__.py
index 580de56..4d0b71f 100644
--- a/test/functional/__init__.py
+++ b/test/functional/__init__.py
@@ -13,9 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import httplib
+from __future__ import print_function
import mock
import os
+from six.moves.urllib.parse import urlparse
import sys
import pickle
import socket
@@ -24,20 +25,23 @@ import eventlet
import eventlet.debug
import functools
import random
-from ConfigParser import ConfigParser, NoSectionError
+
from time import time, sleep
-from httplib import HTTPException
-from urlparse import urlparse
-from nose import SkipTest
from contextlib import closing
from gzip import GzipFile
from shutil import rmtree
from tempfile import mkdtemp
+from unittest2 import SkipTest
+
+from six.moves.configparser import ConfigParser, NoSectionError
+from six.moves import http_client
+from six.moves.http_client import HTTPException
+
from swift.common.middleware.memcache import MemcacheMiddleware
from swift.common.storage_policy import parse_storage_policies, PolicyError
from test import get_config
-from test.functional.swift_test_client import Account, Connection, \
+from test.functional.swift_test_client import Account, Connection, Container, \
ResponseError
# This has the side effect of mocking out the xattr module so that unit tests
# (and in this case, when in-process functional tests are called for) can run
@@ -47,13 +51,13 @@ from test.unit import debug_logger, FakeMemcache
from swift.common import constraints, utils, ring, storage_policy
from swift.common.ring import Ring
from swift.common.wsgi import monkey_patch_mimetools, loadapp
-from swift.common.utils import config_true_value
+from swift.common.utils import config_true_value, split_path
from swift.account import server as account_server
from swift.container import server as container_server
from swift.obj import server as object_server, mem_server as mem_object_server
import swift.proxy.controllers.obj
-httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
+http_client._MAXHEADERS = constraints.MAX_HEADER_COUNT
DEBUG = True
# In order to get the proper blocking behavior of sockets without using
@@ -88,15 +92,16 @@ normalized_urls = None
# If no config was read, we will fall back to old school env vars
swift_test_auth_version = None
swift_test_auth = os.environ.get('SWIFT_TEST_AUTH')
-swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None, '', '']
-swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None, '', '']
-swift_test_tenant = ['', '', '', '', '']
-swift_test_perm = ['', '', '', '', '']
-swift_test_domain = ['', '', '', '', '']
-swift_test_user_id = ['', '', '', '', '']
-swift_test_tenant_id = ['', '', '', '', '']
+swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None, '', '', '']
+swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None, '', '', '']
+swift_test_tenant = ['', '', '', '', '', '']
+swift_test_perm = ['', '', '', '', '', '']
+swift_test_domain = ['', '', '', '', '', '']
+swift_test_user_id = ['', '', '', '', '', '']
+swift_test_tenant_id = ['', '', '', '', '', '']
-skip, skip2, skip3, skip_service_tokens = False, False, False, False
+skip, skip2, skip3, skip_service_tokens, skip_if_no_reseller_admin = \
+ False, False, False, False, False
orig_collate = ''
insecure = False
@@ -105,7 +110,8 @@ orig_hash_path_suff_pref = ('', '')
orig_swift_conf_name = None
in_process = False
-_testdir = _test_servers = _test_coros = None
+_testdir = _test_servers = _test_coros = _test_socks = None
+policy_specified = None
class FakeMemcacheMiddleware(MemcacheMiddleware):
@@ -124,7 +130,7 @@ class InProcessException(BaseException):
def _info(msg):
- print >> sys.stderr, msg
+ print(msg, file=sys.stderr)
def _debug(msg):
@@ -210,7 +216,6 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
for policy in policies:
conf.remove_section(sp_prefix + str(policy.idx))
- policy_specified = os.environ.get('SWIFT_TEST_POLICY')
if policy_specified:
policy_to_test = policies.get_by_name(policy_specified)
if policy_to_test is None:
@@ -282,10 +287,61 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
return obj_sockets
+def _load_encryption(proxy_conf_file, **kwargs):
+ """
+ Load encryption configuration and override proxy-server.conf contents.
+
+ :param proxy_conf_file: Source proxy conf filename
+ :returns: Path to the test proxy conf file to use
+ :raises InProcessException: raised if proxy conf contents are invalid
+ """
+ _debug('Setting configuration for encryption')
+
+ # The global conf dict cannot be used to modify the pipeline.
+ # The pipeline loader requires the pipeline to be set in the local_conf.
+ # If pipeline is set in the global conf dict (which in turn populates the
+ # DEFAULTS options) then it prevents pipeline being loaded into the local
+ # conf during wsgi load_app.
+ # Therefore we must modify the [pipeline:main] section.
+
+ conf = ConfigParser()
+ conf.read(proxy_conf_file)
+ try:
+ section = 'pipeline:main'
+ pipeline = conf.get(section, 'pipeline')
+ pipeline = pipeline.replace(
+ "proxy-logging proxy-server",
+ "keymaster encryption proxy-logging proxy-server")
+ conf.set(section, 'pipeline', pipeline)
+ root_secret = os.urandom(32).encode("base64")
+ conf.set('filter:keymaster', 'encryption_root_secret', root_secret)
+ except NoSectionError as err:
+ msg = 'Error problem with proxy conf file %s: %s' % \
+ (proxy_conf_file, err)
+ raise InProcessException(msg)
+
+ test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
+ with open(test_conf_file, 'w') as fp:
+ conf.write(fp)
+
+ return test_conf_file
+
+
+# Mapping from possible values of the variable
+# SWIFT_TEST_IN_PROCESS_CONF_LOADER
+# to the method to call for loading the associated configuration
+# The expected signature for these methods is:
+# conf_filename_to_use loader(input_conf_filename, **kwargs)
+conf_loaders = {
+ 'encryption': _load_encryption
+}
+
+
def in_process_setup(the_object_server=object_server):
_info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS')
_info('Using object_server class: %s' % the_object_server.__name__)
conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR')
+ show_debug_logs = os.environ.get('SWIFT_TEST_DEBUG_LOGS')
if conf_src_dir is not None:
if not os.path.isdir(conf_src_dir):
@@ -312,6 +368,26 @@ def in_process_setup(the_object_server=object_server):
utils.mkdirs(os.path.join(_testdir, 'sdb1'))
utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
+ # Call the associated method for the value of
+ # 'SWIFT_TEST_IN_PROCESS_CONF_LOADER', if one exists
+ conf_loader_label = os.environ.get(
+ 'SWIFT_TEST_IN_PROCESS_CONF_LOADER')
+ if conf_loader_label is not None:
+ try:
+ conf_loader = conf_loaders[conf_loader_label]
+ _debug('Calling method %s mapped to conf loader %s' %
+ (conf_loader.__name__, conf_loader_label))
+ except KeyError as missing_key:
+ raise InProcessException('No function mapped for conf loader %s' %
+ missing_key)
+
+ try:
+ # Pass-in proxy_conf
+ proxy_conf = conf_loader(proxy_conf)
+ _debug('Now using proxy conf %s' % proxy_conf)
+ except Exception as err: # noqa
+ raise InProcessException(err)
+
swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir)
obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir)
@@ -335,10 +411,13 @@ def in_process_setup(the_object_server=object_server):
orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
utils.validate_hash_conf()
+ global _test_socks
+ _test_socks = []
# We create the proxy server listening socket to get its port number so
# that we can add it as the "auth_port" value for the functional test
# clients.
prolis = eventlet.listen(('localhost', 0))
+ _test_socks.append(prolis)
# The following set of configuration values is used both for the
# functional test frame work and for the various proxy, account, container
@@ -377,13 +456,27 @@ def in_process_setup(the_object_server=object_server):
'service_prefix': 'SERVICE',
# For tempauth middleware. Update reseller_prefix
'reseller_prefix': 'AUTH, SERVICE',
- 'SERVICE_require_group': 'service'
+ 'SERVICE_require_group': 'service',
+ # Reseller admin user (needs reseller_admin_role)
+ 'account6': 'test6',
+ 'username6': 'tester6',
+ 'password6': 'testing6'
})
+ # If an env var explicitly specifies the proxy-server object_post_as_copy
+ # option then use its value, otherwise leave default config unchanged.
+ object_post_as_copy = os.environ.get(
+ 'SWIFT_TEST_IN_PROCESS_OBJECT_POST_AS_COPY')
+ if object_post_as_copy is not None:
+ object_post_as_copy = config_true_value(object_post_as_copy)
+ config['object_post_as_copy'] = str(object_post_as_copy)
+ _debug('Setting object_post_as_copy to %r' % object_post_as_copy)
+
acc1lis = eventlet.listen(('localhost', 0))
acc2lis = eventlet.listen(('localhost', 0))
con1lis = eventlet.listen(('localhost', 0))
con2lis = eventlet.listen(('localhost', 0))
+ _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets
account_ring_path = os.path.join(_testdir, 'account.ring.gz')
with closing(GzipFile(account_ring_path, 'wb')) as f:
@@ -412,23 +505,30 @@ def in_process_setup(the_object_server=object_server):
# Default to only 4 seconds for in-process functional test runs
eventlet.wsgi.WRITE_TIMEOUT = 4
+ def get_logger_name(name):
+ if show_debug_logs:
+ return debug_logger(name)
+ else:
+ return None
+
acc1srv = account_server.AccountController(
- config, logger=debug_logger('acct1'))
+ config, logger=get_logger_name('acct1'))
acc2srv = account_server.AccountController(
- config, logger=debug_logger('acct2'))
+ config, logger=get_logger_name('acct2'))
con1srv = container_server.ContainerController(
- config, logger=debug_logger('cont1'))
+ config, logger=get_logger_name('cont1'))
con2srv = container_server.ContainerController(
- config, logger=debug_logger('cont2'))
+ config, logger=get_logger_name('cont2'))
objsrvs = [
(obj_sockets[index],
the_object_server.ObjectController(
- config, logger=debug_logger('obj%d' % (index + 1))))
+ config, logger=get_logger_name('obj%d' % (index + 1))))
for index in range(len(obj_sockets))
]
- logger = debug_logger('proxy')
+ if show_debug_logs:
+ logger = debug_logger('proxy')
def get_logger(name, *args, **kwargs):
return logger
@@ -442,6 +542,8 @@ def in_process_setup(the_object_server=object_server):
raise InProcessException(e)
nl = utils.NullLogger()
+ global proxy_srv
+ proxy_srv = prolis
prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl)
acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl)
@@ -483,6 +585,7 @@ def get_cluster_info():
# We'll update those constraints based on what the /info API provides, if
# anything.
global cluster_info
+ global config
try:
conn = Connection(config)
conn.authenticate()
@@ -498,7 +601,7 @@ def get_cluster_info():
# Most likely the swift cluster has "expose_info = false" set
# in its proxy-server.conf file, so we'll just do the best we
# can.
- print >>sys.stderr, "** Swift Cluster not exposing /info **"
+ print("** Swift Cluster not exposing /info **", file=sys.stderr)
# Finally, we'll allow any constraint present in the swift-constraints
# section of test.conf to override everything. Note that only those
@@ -510,8 +613,8 @@ def get_cluster_info():
except KeyError:
pass
except ValueError:
- print >>sys.stderr, "Invalid constraint value: %s = %s" % (
- k, test_constraints[k])
+ print("Invalid constraint value: %s = %s" % (
+ k, test_constraints[k]), file=sys.stderr)
eff_constraints.update(test_constraints)
# Just make it look like these constraints were loaded from a /info call,
@@ -521,6 +624,9 @@ def get_cluster_info():
def setup_package():
+
+ global policy_specified
+ policy_specified = os.environ.get('SWIFT_TEST_POLICY')
in_process_env = os.environ.get('SWIFT_TEST_IN_PROCESS')
if in_process_env is not None:
use_in_process = utils.config_true_value(in_process_env)
@@ -529,6 +635,7 @@ def setup_package():
global in_process
+ global config
if use_in_process:
# Explicitly set to True, so barrel on ahead with in-process
# functional test setup.
@@ -540,10 +647,13 @@ def setup_package():
# if the test.conf file is not found, or does not provide a usable
# configuration.
config.update(get_config('func_test'))
- if config:
- in_process = False
- else:
+ if not config:
in_process = True
+ # else... leave in_process value unchanged. It may be that
+ # setup_package is called twice, in which case in_process_setup may
+ # have loaded config before we reach here a second time, so the
+ # existence of config is not reliable to determine that in_process
+ # should be False. Anyway, it's default value is False.
else:
# Explicitly set to False, do not attempt to use in-process
# functional tests, be sure we attempt to read from local
@@ -558,8 +668,8 @@ def setup_package():
in_process_setup(the_object_server=(
mem_object_server if in_mem_obj else object_server))
except InProcessException as exc:
- print >> sys.stderr, ('Exception during in-process setup: %s'
- % str(exc))
+ print(('Exception during in-process setup: %s'
+ % str(exc)), file=sys.stderr)
raise
global web_front_end
@@ -660,6 +770,10 @@ def setup_package():
swift_test_user[4] = config['username5']
swift_test_tenant[4] = config['account5']
swift_test_key[4] = config['password5']
+ if 'username6' in config:
+ swift_test_user[5] = config['username6']
+ swift_test_tenant[5] = config['account6']
+ swift_test_key[5] = config['password6']
for _ in range(5):
swift_test_perm[_] = swift_test_tenant[_] + ':' \
@@ -668,20 +782,19 @@ def setup_package():
global skip
skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]])
if skip:
- print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG'
+ print('SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG', file=sys.stderr)
global skip2
skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]])
if not skip and skip2:
- print >>sys.stderr, \
- 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS' \
- ' DUE TO NO CONFIG FOR THEM'
+ print('SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS '
+ 'DUE TO NO CONFIG FOR THEM', file=sys.stderr)
global skip3
skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]])
if not skip and skip3:
- print >>sys.stderr, \
- 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
+ print('SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS'
+ 'DUE TO NO CONFIG FOR THEM', file=sys.stderr)
global skip_if_not_v3
skip_if_not_v3 = (swift_test_auth_version != '3'
@@ -689,16 +802,42 @@ def setup_package():
swift_test_user[3],
swift_test_key[3]]))
if not skip and skip_if_not_v3:
- print >>sys.stderr, \
- 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3'
+ print('SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3',
+ file=sys.stderr)
global skip_service_tokens
skip_service_tokens = not all([not skip, swift_test_user[4],
swift_test_key[4], swift_test_tenant[4],
swift_test_service_prefix])
if not skip and skip_service_tokens:
- print >>sys.stderr, \
- 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS'
+ print(
+ 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS',
+ file=sys.stderr)
+
+ if policy_specified:
+ policies = FunctionalStoragePolicyCollection.from_info()
+ for p in policies:
+ # policy names are case-insensitive
+ if policy_specified.lower() == p['name'].lower():
+ _info('Using specified policy %s' % policy_specified)
+ FunctionalStoragePolicyCollection.policy_specified = p
+ Container.policy_specified = policy_specified
+ break
+ else:
+ _info(
+ 'SKIPPING FUNCTIONAL TESTS: Failed to find specified policy %s'
+ % policy_specified)
+ raise Exception('Failed to find specified policy %s'
+ % policy_specified)
+
+ global skip_if_no_reseller_admin
+ skip_if_no_reseller_admin = not all([not skip, swift_test_user[5],
+ swift_test_key[5],
+ swift_test_tenant[5]])
+ if not skip and skip_if_no_reseller_admin:
+ print(
+ 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG FOR RESELLER ADMIN',
+ file=sys.stderr)
get_cluster_info()
@@ -708,16 +847,23 @@ def teardown_package():
locale.setlocale(locale.LC_COLLATE, orig_collate)
# clean up containers and objects left behind after running tests
- conn = Connection(config)
- conn.authenticate()
- account = Account(conn, config.get('account', config['username']))
- account.delete_containers()
+ global config
+
+ if config:
+ conn = Connection(config)
+ conn.authenticate()
+ account = Account(conn, config.get('account', config['username']))
+ account.delete_containers()
global in_process
+ global _test_socks
if in_process:
try:
- for server in _test_coros:
+ for i, server in enumerate(_test_coros):
server.kill()
+ if not server.dead:
+ # kill it from the socket level
+ _test_socks[i].close()
except Exception:
pass
try:
@@ -728,6 +874,7 @@ def teardown_package():
orig_hash_path_suff_pref
utils.SWIFT_CONF_FILE = orig_swift_conf_name
constraints.reload_constraints()
+ reset_globals()
class AuthError(Exception):
@@ -745,10 +892,37 @@ parsed = [None, None, None, None, None]
conn = [None, None, None, None, None]
+def reset_globals():
+ global url, token, service_token, parsed, conn, config
+ url = [None, None, None, None, None]
+ token = [None, None, None, None, None]
+ service_token = [None, None, None, None, None]
+ parsed = [None, None, None, None, None]
+ conn = [None, None, None, None, None]
+ if config:
+ config = {}
+
+
def connection(url):
if has_insecure:
- return http_connection(url, insecure=insecure)
- return http_connection(url)
+ parsed_url, http_conn = http_connection(url, insecure=insecure)
+ else:
+ parsed_url, http_conn = http_connection(url)
+
+ orig_request = http_conn.request
+
+ # Add the policy header if policy_specified is set
+ def request_with_policy(method, url, body=None, headers={}):
+ version, account, container, obj = split_path(url, 1, 4, True)
+ if policy_specified and method == 'PUT' and container and not obj \
+ and 'X-Storage-Policy' not in headers:
+ headers['X-Storage-Policy'] = policy_specified
+
+ return orig_request(method, url, body, headers)
+
+ http_conn.request = request_with_policy
+
+ return parsed_url, http_conn
def get_url_token(user_index, os_options):
@@ -899,6 +1073,9 @@ def requires_acls(f):
class FunctionalStoragePolicyCollection(object):
+ # policy_specified is set in __init__.py when tests are being set up.
+ policy_specified = None
+
def __init__(self, policies):
self._all = policies
self.default = None
@@ -940,7 +1117,12 @@ class FunctionalStoragePolicyCollection(object):
p.get(k) != v for k, v in kwargs.items())])
def select(self):
- return random.choice(self)
+ # check that a policy was specified and that it is available
+ # in the current list (i.e., hasn't been excluded of the current list)
+ if self.policy_specified and self.policy_specified in self:
+ return self.policy_specified
+ else:
+ return random.choice(self)
def requires_policies(f):