glusterd: check quorum on restart bricks

While spawning bricks on a glusterd restart the quorum should be checked and
brick shouldn't be started if the volume doesn't meet quorum.

Change-Id: I21bf9055bdf38c53c81138cc204ba05a9ff6444f
BUG: 1298439
Signed-off-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-on: http://review.gluster.org/13236
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

3 files changed, 74 insertions, 5 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-server-quorum.c b/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
index 9f60362b522..7b3f0b79921 100644
--- a/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
+++ b/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
@@ -184,8 +184,8 @@ _is_contributing_to_quorum (gd_quorum_contrib_t contrib)
         return _gf_false;
 }
 
-static gf_boolean_t
-_does_quorum_meet (int active_count, int quorum_count)
+gf_boolean_t
+does_quorum_meet (int active_count, int quorum_count)
 {
         return (active_count >= quorum_count);
 }
@@ -286,7 +286,7 @@ does_gd_meet_server_quorum (xlator_t *this)
         if (ret)
                 goto out;
 
-        if (!_does_quorum_meet (active_count, quorum_count)) {
+        if (!does_quorum_meet (active_count, quorum_count)) {
                 goto out;
         }
 
@@ -384,7 +384,7 @@ glusterd_do_quorum_action ()
                 if (ret)
                         goto unlock;
 
-                if (_does_quorum_meet (active_count, quorum_count))
+                if (does_quorum_meet (active_count, quorum_count))
                         meets = _gf_true;
                 list_for_each_entry (volinfo, &conf->volumes, vol_list) {
                         glusterd_do_volume_quorum_action (this, volinfo, meets);
@@ -396,3 +396,24 @@ unlock:
 out:
         return ret;
 }
+
+/* ret = 1 represents quorum is met or quorum not applicable,
+   ret = 0 represents quorum is not met
+*/
+int
+check_quorum_for_brick_start (glusterd_volinfo_t *volinfo,
+                              gf_boolean_t node_quorum)
+{
+        gf_boolean_t        volume_quorum  =  _gf_false;
+        int                 ret            = 0;
+
+        volume_quorum = glusterd_is_volume_in_server_quorum (volinfo);
+        if (volume_quorum) {
+                if (node_quorum)
+                        ret = 1;
+        } else {
+                ret = 1;
+        }
+        return ret;
+}
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-server-quorum.h b/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
index fc9dd5531e4..ea6a8bd6158 100644
--- a/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
+++ b/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
@@ -35,4 +35,12 @@ glusterd_is_any_volume_in_server_quorum (xlator_t *this);
 
 gf_boolean_t
 does_gd_meet_server_quorum (xlator_t *this);
+
+int
+check_quorum_for_brick_start (glusterd_volinfo_t *volinfo,
+                              gf_boolean_t node_quorum);
+
+gf_boolean_t
+does_quorum_meet (int active_count, int quorum_count);
+
 #endif /* _GLUSTERD_SERVER_QUORUM_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 5f3bd76039d..49b88ed4b13 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -57,6 +57,7 @@
 #include "glusterd-quotad-svc.h"
 #include "glusterd-snapd-svc.h"
 #include "glusterd-bitd-svc.h"
+#include "glusterd-server-quorum.h"
 #include "quota-common-utils.h"
 
 #include "xdr-generic.h"
@@ -4680,9 +4681,23 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
         glusterd_snap_t      *snap           = NULL;
         gf_boolean_t          start_svcs     = _gf_false;
         xlator_t             *this           = NULL;
+        int                   active_count   = 0;
+        int                   quorum_count   = 0;
+        gf_boolean_t          node_quorum    = _gf_false;
 
         this = THIS;
-        GF_ASSERT (this);
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        ret = glusterd_get_quorum_cluster_counts (this, &active_count,
+                                                  &quorum_count);
+        if (ret)
+                goto out;
+
+        if (does_quorum_meet (active_count, quorum_count))
+                node_quorum = _gf_true;
 
         cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
                 if (volinfo->status != GLUSTERD_STATUS_STARTED)
@@ -4693,6 +4708,18 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
                 }
                 gf_msg_debug (this->name, 0, "starting the volume %s",
                         volinfo->volname);
+
+                /* Check the quorum, if quorum is not met, don't start the
+                   bricks
+                */
+                ret = check_quorum_for_brick_start (volinfo, node_quorum);
+                if (ret == 0) {
+                        gf_msg (this->name, GF_LOG_INFO, 0,
+                                GD_MSG_SERVER_QUORUM_NOT_MET, "Skipping brick "
+                                "restart for volume %s as quorum is not met",
+                                volinfo->volname);
+                        continue;
+                }
                 cds_list_for_each_entry (brickinfo, &volinfo->bricks,
                                          brick_list) {
                         glusterd_brick_start (volinfo, brickinfo, _gf_false);
@@ -4703,6 +4730,18 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
                 cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
                         if (volinfo->status != GLUSTERD_STATUS_STARTED)
                                 continue;
+                        /* Check the quorum, if quorum is not met, don't start the
+                           bricks
+                        */
+                        ret = check_quorum_for_brick_start (volinfo,
+                                                            node_quorum);
+                        if (ret == 0) {
+                                gf_msg (this->name, GF_LOG_INFO, 0,
+                                        GD_MSG_SERVER_QUORUM_NOT_MET, "Skipping"
+                                        " brick restart for volume %s as "
+                                        "quorum is not met", volinfo->volname);
+                                continue;
+                        }
                         if (start_svcs == _gf_false) {
                                 start_svcs = _gf_true;
                                 glusterd_svcs_manager (volinfo);
@@ -4718,6 +4757,7 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
                 }
         }
 
+out:
         return ret;
 }