adding library and test case for examine gluster

This library is helpfull for diagnosing output of gluster and heketi
and it compare both the outputs and the test case verify this feature
is working properly.

Change-Id: Ice607003f32db2bce2f540a7651024b66b071634

2 files changed, 83 insertions, 0 deletions

diff --git a/cns-libs/cnslibs/common/heketi_ops.py b/cns-libs/cnslibs/common/heketi_ops.py
index 2d1b7385..a1248687 100644
--- a/cns-libs/cnslibs/common/heketi_ops.py
+++ b/cns-libs/cnslibs/common/heketi_ops.py
@@ -3,6 +3,7 @@ import json
 from glusto.core import Glusto as g
 
 from cnslibs.common import exceptions
+from cnslibs.common import heketi_version
 from cnslibs.common.utils import parse_prometheus_data
 
 
@@ -1455,3 +1456,40 @@ def get_heketi_metrics(heketi_client_node, heketi_server_url,
     if prometheus_format:
         return out.strip()
     return parse_prometheus_data(out)
+
+
+def heketi_examine_gluster(heketi_client_node, heketi_server_url):
+    """Execute heketi command to examine output from gluster servers.
+
+    Args:
+        - heketi_client_node (str): Node where we want to run our commands.
+        - heketi_server_url (str): This is a heketi server url.
+
+    Raises:
+        NotImplementedError: if heketi version is not expected
+        exceptions.ExecutionError: if command fails.
+
+    Returns:
+        dictionary: if successful
+    """
+
+    version = heketi_version.get_heketi_version(heketi_client_node)
+    if version < '8.0.0-7':
+        msg = ("heketi-client package %s does not support server state examine"
+               " gluster" % version.v_str)
+        g.log.error(msg)
+        raise NotImplementedError(msg)
+
+    heketi_server_url, json_arg, secret, user = _set_heketi_global_flags(
+        heketi_server_url)
+    # output is always json-like and we do not need to provide "--json" CLI arg
+    cmd = ("heketi-cli server state examine gluster -s %s %s %s"
+           % (heketi_server_url, user, secret))
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = "failed to examine gluster with following error: %s" % err
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+
+    return json.loads(out)
diff --git a/tests/functional/common/heketi/test_server_state_examine_gluster.py b/tests/functional/common/heketi/test_server_state_examine_gluster.py
new file mode 100644
index 00000000..5b904e8f
--- /dev/null
+++ b/tests/functional/common/heketi/test_server_state_examine_gluster.py
@@ -0,0 +1,45 @@
+from cnslibs.cns import cns_baseclass
+from cnslibs.common import heketi_ops
+from cnslibs.common import heketi_version
+from cnslibs.common import openshift_ops
+
+
+class TestHeketiServerStateExamineGluster(cns_baseclass.BaseClass):
+
+    def setUp(self):
+        self.node = self.ocp_master_node[0]
+        version = heketi_version.get_heketi_version(self.heketi_client_node)
+        if version < '8.0.0-7':
+            self.skipTest("heketi-client package %s does not support server "
+                          "state examine gluster" % version.v_str)
+
+    def test_volume_inconsistencies(self):
+        # Examine Gluster cluster and Heketi that there is no inconsistencies
+        out = heketi_ops.heketi_examine_gluster(
+            self.heketi_client_node, self.heketi_server_url)
+        if ("heketi volume list matches with volume list of all nodes"
+                not in out['report']):
+            self.skipTest(
+                "heketi and Gluster are inconsistent to each other")
+
+        # create volume
+        vol = heketi_ops.heketi_volume_create(
+            self.heketi_client_node, self.heketi_server_url, 1, json=True)
+        self.addCleanup(
+            heketi_ops.heketi_volume_delete, self.heketi_client_node,
+            self.heketi_server_url, vol['id'])
+
+        # delete volume from gluster cluster directly
+        openshift_ops.cmd_run_on_gluster_pod_or_node(
+            self.node,
+            "gluster vol stop %s force --mode=script" % vol['name'])
+        openshift_ops.cmd_run_on_gluster_pod_or_node(
+            self.node,
+            "gluster vol delete %s --mode=script" % vol['name'])
+
+        # verify that heketi is reporting inconsistencies
+        out = heketi_ops.heketi_examine_gluster(
+            self.heketi_client_node, self.heketi_server_url)
+        self.assertNotIn(
+            "heketi volume list matches with volume list of all nodes",
+            out['report'])