glusterd: quorum validatation in glusterd syncop framework

Previously glusterd was not checking quorum validation in syncop framework.
So when there is loss in quorum then few operation (for eg. add-brick,
remove-brick, volume set) which is based on syncop framework passed
successfully with out doing quorum validation check.

With this change it will do quorum validation in syncop framework and it will
block all operation (except volume set <quorum options> and "volume reset all"
commands) when there is loss in quorum.

Change-Id: I4c2ef16728d55c98a228bb86795023d9c1f4e9fb
BUG: 1177132
Signed-off-by: Gaurav Kumar Garg <ggarg@redhat.com>
Reviewed-on: http://review.gluster.org/9349
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Krishnan Parthasarathi <kparthas@redhat.com>

6 files changed, 181 insertions, 143 deletions

diff --git a/tests/bugs/glusterd/bug-1177132-quorum-calculation-fix.t b/tests/bugs/glusterd/bug-1177132-quorum-calculation-fix.t
deleted file mode 100755
index e10fd193f5d..00000000000
--- a/tests/bugs/glusterd/bug-1177132-quorum-calculation-fix.t
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/../../include.rc
-. $(dirname $0)/../../volume.rc
-. $(dirname $0)/../../cluster.rc
-
-cleanup;
-
-TEST launch_cluster 2;
-TEST $CLI_1 peer probe $H2;
-
-EXPECT_WITHIN $PROBE_TIMEOUT 1 peer_count
-
-# Lets create the volume and set quorum type as a server
-TEST $CLI_1 volume create $V0 $H1:$B1/${V0}0 $H2:$B2/${V0}1
-TEST $CLI_1 volume set $V0 cluster.server-quorum-type server
-
-# Start the volume
-TEST $CLI_1 volume start $V0
-
-# Set the quorum ratio to 80% which means in a two node cluster if one node is
-# down quorum shouldn't be met and operations which goes through quorum
-# validation should fail
-TEST $CLI_1 volume set all cluster.server-quorum-ratio 80
-
-# Bring down one glusterd instance
-TEST kill_glusterd 2
-
-# Now execute a command which goes through op state machine and it should fail
-TEST ! $CLI_1 volume profile $V0 start
-
-# Bring back the glusterd instance
-TEST $glusterd_2
-
-EXPECT_WITHIN $PROBE_TIMEOUT 1 peer_count;
-
-# Now re-execute the same profile command and this time it should succeed
-TEST $CLI_1 volume profile $V0 start
-
-cleanup;
diff --git a/tests/bugs/glusterd/bug-1177132-quorum-validation.t b/tests/bugs/glusterd/bug-1177132-quorum-validation.t
new file mode 100644
index 00000000000..57aec5ccf57
--- /dev/null
+++ b/tests/bugs/glusterd/bug-1177132-quorum-validation.t
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Test case for quorum validation in glusterd for syncop framework
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../cluster.rc
+
+
+cleanup;
+
+TEST launch_cluster 2
+
+TEST $CLI_1 peer probe $H2;
+EXPECT_WITHIN $PROBE_TIMEOUT 1 peer_count
+
+# Lets create the volume and set quorum type as a server
+TEST $CLI_1 volume create $V0 $H1:$B1/${V0}0 $H2:$B2/${V0}1
+TEST $CLI_1 volume set $V0 cluster.server-quorum-type server
+
+# Start the volume
+TEST $CLI_1 volume start $V0
+
+# Set quorum ratio 52. means 52 % or more than 52% nodes of total available node
+# should be available for performing volume operation.
+# i.e. Server-side quorum is met if the number of nodes that are available is
+# greater than or equal to 'quorum-ratio' times the number of nodes in the
+# cluster
+
+TEST $CLI_1 volume set all cluster.server-quorum-ratio 52
+
+# Bring down 2nd glusterd
+TEST kill_glusterd 2
+
+# Now quorum is not meet. Add-brick, Remove-brick, volume-set command
+#(Command based on syncop framework)should fail
+TEST ! $CLI_1 volume add-brick $V0 $H1:$B1/${V0}1
+TEST ! $CLI_1 volume remove-brick $V0 $H1:$B1/${V0}0 start
+TEST ! $CLI_1 volume set $V0 barrier enable
+
+# Now execute a command which goes through op state machine and it should fail
+TEST ! $CLI_1 volume profile $V0 start
+
+# Volume set all command and volume reset all command should be successful
+TEST $CLI_1 volume set all cluster.server-quorum-ratio 80
+TEST $CLI_1 volume reset all
+
+# Bring back 2nd glusterd
+TEST $glusterd_2
+
+# After 2nd glusterd come back, there will be 2 nodes in a clusater
+EXPECT_WITHIN $PROBE_TIMEOUT 1 peer_count;
+
+# Now quorum is meet.
+# Add-brick, Remove-brick, volume-set command should success
+TEST  $CLI_1 volume add-brick $V0 $H2:$B2/${V0}2
+TEST  $CLI_1 volume remove-brick $V0 $H2:$B2/${V0}2 start
+TEST  $CLI_1 volume set $V0 barrier enable
+TEST  $CLI_1 volume remove-brick $V0 $H2:$B2/${V0}2 stop
+
+# Now re-execute the same profile command and this time it should succeed
+TEST $CLI_1 volume profile $V0 start
+
+cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index a7d1095f4e9..9352b9accab 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -3390,108 +3390,6 @@ out:
         return ret;
 }
 
-gf_boolean_t
-glusterd_is_get_op (xlator_t *this, glusterd_op_t op, dict_t *dict)
-{
-        char            *key = NULL;
-        char            *volname = NULL;
-        int             ret = 0;
-
-        if (op == GD_OP_STATUS_VOLUME)
-                return _gf_true;
-
-        if (op == GD_OP_SET_VOLUME) {
-                //check for set volume help
-                ret = dict_get_str (dict, "volname", &volname);
-                if (volname &&
-                    ((strcmp (volname, "help") == 0) ||
-                     (strcmp (volname, "help-xml") == 0))) {
-                        ret = dict_get_str (dict, "key1", &key);
-                        if (ret < 0)
-                                return _gf_true;
-                }
-        }
-
-        return _gf_false;
-}
-
-gf_boolean_t
-glusterd_is_op_quorum_validation_required (xlator_t *this, glusterd_op_t op,
-                                           dict_t *dict)
-{
-        gf_boolean_t    required = _gf_true;
-        char            *key = NULL;
-        char            *key_fixed = NULL;
-        int             ret = -1;
-
-        if (glusterd_is_get_op (this, op, dict)) {
-                required = _gf_false;
-                goto out;
-        }
-        if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
-                goto out;
-        if (op == GD_OP_SET_VOLUME)
-                ret = dict_get_str (dict, "key1", &key);
-        else if (op == GD_OP_RESET_VOLUME)
-                ret = dict_get_str (dict, "key", &key);
-        if (ret)
-                goto out;
-        ret = glusterd_check_option_exists (key, &key_fixed);
-        if (ret <= 0)
-                goto out;
-        if (key_fixed)
-                key = key_fixed;
-        if (glusterd_is_quorum_option (key))
-                required = _gf_false;
-out:
-        GF_FREE (key_fixed);
-        return required;
-}
-
-/* This function should not be used when the quorum validation needs to happen
- * on non-global peer list */
-static int
-glusterd_op_validate_quorum (xlator_t *this, glusterd_op_t op,
-                             dict_t *dict, char **op_errstr)
-{
-        int                      ret     = 0;
-        char                    *volname = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    *errstr  = NULL;
-
-        errstr = "Quorum not met. Volume operation not allowed.";
-        if (!glusterd_is_op_quorum_validation_required (this, op, dict))
-                goto out;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
-
-        /* Passing NULL implies quorum calculation will happen on global peer
-         * list */
-        if (does_gd_meet_server_quorum (this, NULL, _gf_false)) {
-                ret = 0;
-                goto out;
-        }
-
-        if (glusterd_is_volume_in_server_quorum (volinfo)) {
-                ret = -1;
-                *op_errstr = gf_strdup (errstr);
-                goto out;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
 static int
 glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 {
@@ -3530,7 +3428,7 @@ glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
                 goto out;
         }
 
-        ret = glusterd_op_validate_quorum (this, op, dict, &op_errstr);
+        ret = glusterd_validate_quorum (this, op, dict, &op_errstr);
         if (ret) {
                 gf_msg (this->name, GF_LOG_CRITICAL, 0,
                         GD_MSG_SERVER_QUORUM_NOT_MET,
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index 1de9e4603a2..f37aa483a00 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -20,6 +20,7 @@
 #include "glusterd-utils.h"
 #include "glusterd-locks.h"
 #include "glusterd-snapshot-utils.h"
+#include "glusterd-messages.h"
 
 extern glusterd_op_info_t opinfo;
 
@@ -1211,6 +1212,8 @@ gd_stage_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
         dict_t                 *aggr_dict       = NULL;
 
         this = THIS;
+        GF_ASSERT (this);
+
         rsp_dict = dict_new ();
         if (!rsp_dict)
                 goto out;
@@ -1221,6 +1224,14 @@ gd_stage_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
         else
                 aggr_dict = op_ctx;
 
+        ret = glusterd_validate_quorum (this, op, req_dict, op_errstr);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_CRITICAL, 0,
+                        GD_MSG_SERVER_QUORUM_NOT_MET,
+                        "Server quorum not met. Rejecting operation.");
+                goto out;
+        }
+
         ret = glusterd_op_stage_validate (op, req_dict, op_errstr, rsp_dict);
         if (ret) {
                 hostname = "localhost";
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index d829676228c..5c2d6f23309 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1096,6 +1096,107 @@ out:
 }
 #endif
 
+gf_boolean_t
+glusterd_is_get_op (xlator_t *this, glusterd_op_t op, dict_t *dict)
+{
+        char            *key = NULL;
+        char            *volname = NULL;
+        int             ret = 0;
+
+        if (op == GD_OP_STATUS_VOLUME)
+                return _gf_true;
+
+        if (op == GD_OP_SET_VOLUME) {
+                /*check for set volume help*/
+                ret = dict_get_str (dict, "volname", &volname);
+                if (volname &&
+                    ((strcmp (volname, "help") == 0) ||
+                     (strcmp (volname, "help-xml") == 0))) {
+                        ret = dict_get_str (dict, "key1", &key);
+                        if (ret < 0)
+                                return _gf_true;
+                }
+        }
+        return _gf_false;
+}
+
+gf_boolean_t
+glusterd_is_quorum_validation_required (xlator_t *this, glusterd_op_t op,
+                                        dict_t *dict)
+{
+        gf_boolean_t    required = _gf_true;
+        char            *key = NULL;
+        char            *key_fixed = NULL;
+        int             ret = -1;
+
+        if (glusterd_is_get_op (this, op, dict)) {
+                required = _gf_false;
+                goto out;
+        }
+        if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
+                goto out;
+        if (op == GD_OP_SET_VOLUME)
+                ret = dict_get_str (dict, "key1", &key);
+        else if (op == GD_OP_RESET_VOLUME)
+                ret = dict_get_str (dict, "key", &key);
+        if (ret)
+                goto out;
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0)
+                goto out;
+        if (key_fixed)
+                key = key_fixed;
+        if (glusterd_is_quorum_option (key))
+                required = _gf_false;
+out:
+        GF_FREE (key_fixed);
+        return required;
+}
+
+/* This function should not be used when the quorum validation needs to happen
+ * on non-global peer list */
+int
+glusterd_validate_quorum (xlator_t *this, glusterd_op_t op,
+                             dict_t *dict, char **op_errstr)
+{
+        int                      ret     = 0;
+        char                    *volname = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *errstr  = NULL;
+
+        errstr = "Quorum not met. Volume operation not allowed.";
+        if (!glusterd_is_quorum_validation_required (this, op, dict))
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Passing NULL implies quorum calculation will happen on global peer
+         * list */
+        if (does_gd_meet_server_quorum (this, NULL, _gf_false)) {
+                ret = 0;
+                goto out;
+        }
+
+        if (glusterd_is_volume_in_server_quorum (volinfo)) {
+                ret = -1;
+                *op_errstr = gf_strdup (errstr);
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
 int
 glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
                                         uuid_t volume_id, char **op_errstr,
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 274e49e1c22..63be397aeac 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -138,6 +138,10 @@ glusterd_service_stop(const char *service, char *pidfile, int sig,
 int
 glusterd_get_next_available_brickid (glusterd_volinfo_t *volinfo);
 
+int
+glusterd_validate_quorum (xlator_t *this, glusterd_op_t op, dict_t *dict,
+                          char **op_errstr);
+
 int32_t
 glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo);