From 5fe05c3d09126453423a3d2ee97a4586ec6592c1 Mon Sep 17 00:00:00 2001 From: Valerii Ponomarov Date: Thu, 21 Mar 2019 16:04:57 +0530 Subject: Add py3 support to our tox config and fix several incompatibilities Required steps to run test cases using py3: 1) Install py3 2) Install 'tox' package using "pip3" 3) Run test cases using tox package installed via pip3 Note that full py3 support is not tested yet. It only allows you to run it on py3 not providing any guarantees. Example: $ python3 -m tox -e functional3 -- \ glusto -c /path/to/the/config/file.yaml \ '--pytest=-v -rsx tests -k test_glusterblock_logs' Change-Id: I610faac0b75364c401734b6f892649893ca2320e --- .../openshiftstoragelibs/baseclass.py | 8 ++++---- .../openshiftstoragelibs/gluster_ops.py | 9 +++++++-- .../openshiftstoragelibs/heketi_ops.py | 7 ++++++- .../openshiftstoragelibs/heketi_version.py | 2 +- .../openshiftstoragelibs/naming.py | 7 ++++++- .../openshiftstoragelibs/openshift_ops.py | 21 ++++++++++++++------- openshift-storage-libs/setup.py | 2 +- ...test_create_distributed_replica_heketi_volume.py | 4 ++-- .../heketi/test_heketi_device_operations.py | 7 ++++++- tests/functional/heketi/test_node_enable_disable.py | 7 ++++++- tests/functional/heketi/test_volume_multi_req.py | 10 +++++----- tox.ini | 19 +++++++++++++++++++ 12 files changed, 77 insertions(+), 26 deletions(-) diff --git a/openshift-storage-libs/openshiftstoragelibs/baseclass.py b/openshift-storage-libs/openshiftstoragelibs/baseclass.py index c7891608..375ab7c0 100644 --- a/openshift-storage-libs/openshiftstoragelibs/baseclass.py +++ b/openshift-storage-libs/openshiftstoragelibs/baseclass.py @@ -43,11 +43,11 @@ class BaseClass(unittest.TestCase): # Initializes OCP config variables cls.ocp_servers_info = g.config['ocp_servers'] - cls.ocp_master_node = g.config['ocp_servers']['master'].keys() + cls.ocp_master_node = list(g.config['ocp_servers']['master'].keys()) cls.ocp_master_node_info = g.config['ocp_servers']['master'] - cls.ocp_client = g.config['ocp_servers']['client'].keys() + cls.ocp_client = list(g.config['ocp_servers']['client'].keys()) cls.ocp_client_info = g.config['ocp_servers']['client'] - cls.ocp_nodes = g.config['ocp_servers']['nodes'].keys() + cls.ocp_nodes = list(g.config['ocp_servers']['nodes'].keys()) cls.ocp_nodes_info = g.config['ocp_servers']['nodes'] # Initializes storage project config variables @@ -65,7 +65,7 @@ class BaseClass(unittest.TestCase): cls.heketi_cli_user = heketi_config['heketi_cli_user'] cls.heketi_cli_key = heketi_config['heketi_cli_key'] - cls.gluster_servers = g.config['gluster_servers'].keys() + cls.gluster_servers = list(g.config['gluster_servers'].keys()) cls.gluster_servers_info = g.config['gluster_servers'] cls.storage_classes = openshift_config['dynamic_provisioning'][ diff --git a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py index 8ac95d82..798e3c2a 100644 --- a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py @@ -1,6 +1,11 @@ -import time -import json +try: + # py2/3 + import simplejson as json +except ImportError: + # py2 + import json import re +import time from glusto.core import Glusto as g from glustolibs.gluster.block_ops import block_list diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py index f97dadc4..b0ecc7c7 100644 --- a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py @@ -1,4 +1,9 @@ -import json +try: + # py2/3 + import simplejson as json +except ImportError: + # py2 + import json from glusto.core import Glusto as g diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_version.py b/openshift-storage-libs/openshiftstoragelibs/heketi_version.py index 0da81176..c199d9cf 100644 --- a/openshift-storage-libs/openshiftstoragelibs/heketi_version.py +++ b/openshift-storage-libs/openshiftstoragelibs/heketi_version.py @@ -75,7 +75,7 @@ def _get_heketi_server_version_str(ocp_client_node=None): Raises: 'exceptions.ExecutionError' if failed to get version """ if not ocp_client_node: - ocp_client_node = g.config["ocp_servers"]["client"].keys()[0] + ocp_client_node = list(g.config["ocp_servers"]["client"].keys())[0] get_package_version_cmd = ( "rpm -q heketi --queryformat '%{version}-%{release}\n' | " "cut -d '.' -f 1,2,3") diff --git a/openshift-storage-libs/openshiftstoragelibs/naming.py b/openshift-storage-libs/openshiftstoragelibs/naming.py index b44559ad..3d97c3e0 100644 --- a/openshift-storage-libs/openshiftstoragelibs/naming.py +++ b/openshift-storage-libs/openshiftstoragelibs/naming.py @@ -7,7 +7,12 @@ import re # we only use lowercase here because kubernetes requires # names to be lowercase or digits, so that is our default -UNIQUE_CHARS = (string.lowercase + string.digits) +try: + # py2 + UNIQUE_CHARS = (string.lowercase + string.digits) +except AttributeError: + # py3 + UNIQUE_CHARS = (string.ascii_lowercase + string.digits) def make_unique_label(prefix=None, suffix=None, sep='-', diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py index 295dc42b..ae0de983 100644 --- a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py @@ -4,7 +4,12 @@ Various utility functions for interacting with OCP/OpenShift. """ import base64 -import json +try: + # py2/3 + import simplejson as json +except ImportError: + # py2 + import json import re import types @@ -111,7 +116,7 @@ def get_ocp_gluster_pod_names(ocp_node): get_ocp_gluster_pod_names(ocp_node) """ - pod_names = oc_get_pods(ocp_node).keys() + pod_names = list(oc_get_pods(ocp_node).keys()) return [pod for pod in pod_names if pod.startswith('glusterfs-')] @@ -269,6 +274,7 @@ def oc_create_secret(hostname, secret_name_prefix="autotests-secret-", Returns: name of a secret """ secret_name = "%s-%s" % (secret_name_prefix, utils.get_random_str()) + data_key = data_key.encode('utf-8') secret_data = json.dumps({ "apiVersion": "v1", "data": {"key": base64.b64encode(data_key)}, @@ -527,11 +533,12 @@ def oc_get_custom_resource(ocp_node, rtype, custom, name=None, selector=None, return [] if name: - return filter(None, map(str.strip, (out.strip()).split(' '))) + return list(filter(None, map(str.strip, (out.strip()).split(' ')))) else: out_list = [] for line in (out.strip()).split('\n'): - out_list.append(filter(None, map(str.strip, line.split(' ')))) + out_list.append( + list(filter(None, map(str.strip, line.split(' '))))) return out_list @@ -802,7 +809,7 @@ def cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd, gluster_node=None): "Could not find Gluster PODs with node filter as " "'%s'." % gluster_node) else: - gluster_pod_names = gluster_pods.keys() + gluster_pod_names = list(gluster_pods.keys()) for gluster_pod_name in gluster_pod_names: try: @@ -819,7 +826,7 @@ def cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd, gluster_node=None): if gluster_node: g_hosts = [gluster_node] else: - g_hosts = g.config.get("gluster_servers", {}).keys() + g_hosts = list(g.config.get("gluster_servers", {}).keys()) for g_host in g_hosts: try: return command.cmd_run(cmd, hostname=g_host) @@ -887,7 +894,7 @@ def get_gluster_blockvol_info_by_pvc_name(ocp_node, heketi_server_url, r':.metadata.annotations."gluster\.org\/volume\-id",' r':.spec.claimRef.name | grep "%s"' % pvc_name) out = command.cmd_run(get_block_vol_data_cmd, hostname=ocp_node) - parsed_out = filter(None, map(str.strip, out.split(" "))) + parsed_out = list(filter(None, map(str.strip, out.split(" ")))) assert len(parsed_out) == 3, "Expected 3 fields in following: %s" % out block_vol_name, block_vol_id = parsed_out[:2] diff --git a/openshift-storage-libs/setup.py b/openshift-storage-libs/setup.py index 3e528cbf..ab6df205 100644 --- a/openshift-storage-libs/setup.py +++ b/openshift-storage-libs/setup.py @@ -23,7 +23,7 @@ setup( 'Topic :: Software Development :: Testing' ], install_requires=['glusto', 'ddt', 'mock', 'rtyaml', 'jsondiff', 'six', - 'prometheus_client>=0.4.2'], + 'simplejson', 'prometheus_client>=0.4.2'], dependency_links=[ 'http://github.com/loadtheaccumulator/glusto/tarball/master#egg=glusto' ], diff --git a/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py b/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py index 04bce628..8a3beb29 100644 --- a/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py +++ b/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py @@ -24,8 +24,8 @@ class TestHeketiVolume(BaseClass): def setUp(self): super(TestHeketiVolume, self).setUp() - self.master_node = g.config['ocp_servers']['master'].keys()[0] - self.gluster_node = g.config["gluster_servers"].keys()[0] + self.master_node = list(g.config['ocp_servers']['master'].keys())[0] + self.gluster_node = list(g.config["gluster_servers"].keys())[0] def _get_free_space(self): """Get free space in each heketi device""" diff --git a/tests/functional/heketi/test_heketi_device_operations.py b/tests/functional/heketi/test_heketi_device_operations.py index bec1d01f..76736ecf 100644 --- a/tests/functional/heketi/test_heketi_device_operations.py +++ b/tests/functional/heketi/test_heketi_device_operations.py @@ -1,4 +1,9 @@ -import json +try: + # py2/3 + import simplejson as json +except ImportError: + # py2 + import json import ddt from glusto.core import Glusto as g diff --git a/tests/functional/heketi/test_node_enable_disable.py b/tests/functional/heketi/test_node_enable_disable.py index dcd2f7b4..e1e416fc 100644 --- a/tests/functional/heketi/test_node_enable_disable.py +++ b/tests/functional/heketi/test_node_enable_disable.py @@ -1,5 +1,10 @@ """Test cases to disable and enable node in heketi.""" -import json +try: + # py2/3 + import simplejson as json +except ImportError: + # py2 + import json from glusto.core import Glusto as g diff --git a/tests/functional/heketi/test_volume_multi_req.py b/tests/functional/heketi/test_volume_multi_req.py index 3445a8a4..9fcaeb49 100644 --- a/tests/functional/heketi/test_volume_multi_req.py +++ b/tests/functional/heketi/test_volume_multi_req.py @@ -219,7 +219,7 @@ class TestVolumeMultiReq(BaseClass): 'wait for volume count to settle timed out') def _count_vols(self): - ocp_node = g.config['ocp_servers']['master'].keys()[0] + ocp_node = list(g.config['ocp_servers']['master'].keys())[0] return len(_heketi_vols(ocp_node, self.heketi_server_url)) def test_simple_serial_vol_create(self): @@ -229,7 +229,7 @@ class TestVolumeMultiReq(BaseClass): # TODO A nice thing to add to this test would be to also verify # the gluster volumes also exist. tname = make_unique_label(extract_method_name(self.id())) - ocp_node = g.config['ocp_servers']['master'].keys()[0] + ocp_node = list(g.config['ocp_servers']['master'].keys())[0] # deploy a temporary storage class sc = build_storage_class( name=tname, @@ -290,7 +290,7 @@ class TestVolumeMultiReq(BaseClass): """ self.addCleanup(self.wait_to_settle) tname = make_unique_label(extract_method_name(self.id())) - ocp_node = g.config['ocp_servers']['master'].keys()[0] + ocp_node = list(g.config['ocp_servers']['master'].keys())[0] # deploy a temporary storage class sc = build_storage_class( name=tname, @@ -345,7 +345,7 @@ class TestVolumeMultiReq(BaseClass): """ self.addCleanup(self.wait_to_settle) tname = make_unique_label(extract_method_name(self.id())) - ocp_node = g.config['ocp_servers']['master'].keys()[0] + ocp_node = list(g.config['ocp_servers']['master'].keys())[0] # deploy a temporary storage class sc = build_storage_class( name=tname, @@ -389,7 +389,7 @@ class TestVolumeMultiReq(BaseClass): """ self.addCleanup(self.wait_to_settle) tname = make_unique_label(extract_method_name(self.id())) - ocp_node = g.config['ocp_servers']['master'].keys()[0] + ocp_node = list(g.config['ocp_servers']['master'].keys())[0] # deploy a temporary storage class sc = build_storage_class( name=tname, diff --git a/tox.ini b/tox.ini index 634830ac..30eb9eca 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ commands = commands = {[testenv]commands} pip install \ + simplejson \ mock \ rtyaml \ ddt \ @@ -32,6 +33,24 @@ commands = --editable=file:///{toxinidir}/openshift-storage-libs {posargs:bash -c "echo 'No commands have been specified. Exiting.'; exit 1"} +[testenv:functional3] +basepython = python3 +commands = + find . -type f -name "*.py[c|o]" -delete + python3 -m pip install --upgrade pip>=9.0.0 setuptools wheel + pip3 install \ + simplejson \ + mock \ + rtyaml \ + ddt \ + prometheus_client>=0.4.2 \ + git+git://github.com/loadtheaccumulator/glusto.git@python3_port1 \ + "git+git://github.com/gluster/glusto-tests.git#egg=glustolibs-gluster&subdirectory=glustolibs-gluster" \ + "git+git://github.com/gluster/glusto-tests.git#egg=glustolibs-io&subdirectory=glustolibs-io" \ + "git+git://github.com/gluster/glusto-tests.git#egg=glustolibs-misc&subdirectory=glustolibs-misc" \ + --editable=file:///{toxinidir}/openshift-storage-libs + {posargs:bash -c "echo 'No commands have been specified. Exiting.'; exit 1"} + [testenv:venv] commands = {posargs} -- cgit