glusterd/add-brick: change add-brick implementation to v3 framework

add-brick commit first happens on local node and followed by peers.
As part of the commit of local-host glusterd will send the updated
volfiles to the clients connected to the local-host even before the
commit of peers happen. If any of the newly added brick was hosted
by any peer, that brick won't be started when client (connected to
local-host) try to send fops.

By changing to v3 framework we can send post validate ops
after commit operation that helps to send volfile fetch request only
after completing commits on all nodes.

Change-Id: Ib7312e01143326128c010c11fc2ed206f37409ad
BUG: 1263549
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/12237
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

2 files changed, 134 insertions, 17 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index 8772dd1d108..fa652ac60c1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -14,10 +14,12 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-geo-rep.h"
 #include "glusterd-store.h"
+#include "glusterd-mgmt.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-svc-helper.h"
 #include "glusterd-messages.h"
+#include "glusterd-server-quorum.h"
 #include "run.h"
 #include <sys/signal.h>
 
@@ -412,11 +414,16 @@ __glusterd_handle_add_brick (rpcsvc_request_t *req)
         int32_t                         replica_count = 0;
         int32_t                         stripe_count = 0;
         int                             type = 0;
+        glusterd_conf_t                 *conf = NULL;
+
         this = THIS;
         GF_ASSERT(this);
 
         GF_ASSERT (req);
 
+        conf = this->private;
+        GF_ASSERT (conf);
+
         ret = xdr_to_generic (req->msg[0], &cli_req,
                               (xdrproc_t)xdr_gf_cli_req);
         if (ret < 0) {
@@ -635,7 +642,17 @@ brick_val:
                 }
         }
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
+        if (conf->op_version <= GD_OP_VERSION_3_7_5) {
+                gf_msg_debug (this->name, 0, "The cluster is operating at "
+                          "version less than or equal to %d. Falling back "
+                          "to syncop framework.",
+                          GD_OP_VERSION_3_7_5);
+                ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
+        } else {
+                ret = glusterd_mgmt_v3_initiate_all_phases (req,
+                                                            GD_OP_ADD_BRICK,
+                                                            dict);
+        }
 
 out:
         if (ret) {
@@ -1318,14 +1335,14 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         volinfo->subvol_count = (volinfo->brick_count /
                                  volinfo->dist_leaf_count);
 
-        ret = glusterd_create_volfiles (volinfo);
-        if (ret)
-                goto out;
-
         ret = 0;
         if (GLUSTERD_STATUS_STARTED != volinfo->status)
                 goto out;
 
+        ret = generate_brick_volfiles (volinfo);
+        if (ret)
+                goto out;
+
         brick_list = gf_strdup (bricks);
         free_ptr2 = brick_list;
         i = 1;
@@ -1396,8 +1413,16 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                               _glusterd_restart_gsync_session, &param);
         }
         volinfo->caps = caps;
-
-        ret = glusterd_fetchspec_notify (this);
+        if (conf->op_version <= GD_OP_VERSION_3_7_5) {
+               ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        } else {
+                /*
+                 * The cluster is operating at version greater than
+                 * gluster-3.7.5. So no need to sent volfile fetch
+                 * request in commit phase, the same will be done
+                 * in post validate phase with v3 framework.
+                 */
+        }
 
 out:
         GF_FREE (free_ptr1);
@@ -1523,6 +1548,28 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         if (ret)
                 goto out;
 
+        if (conf->op_version > GD_OP_VERSION_3_7_5 &&
+            is_origin_glusterd (dict)) {
+                ret = glusterd_validate_quorum (this, GD_OP_ADD_BRICK, dict,
+                                                op_errstr);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
+                                GD_MSG_SERVER_QUORUM_NOT_MET,
+                                "Server quorum not met. Rejecting operation.");
+                        goto out;
+                }
+        } else {
+                /* Case 1: conf->op_version <= GD_OP_VERSION_3_7_5
+                 *         in this case the add-brick is running
+                 *         syncop framework that will do a quorum
+                 *         check by default
+                 * Case 2: We don't need to do quorum check on every
+                 *         node, only originator glusterd need to
+                 *         check for quorum
+                 * So nothing need to be done in else
+                 */
+        }
+
         if (glusterd_is_defrag_on(volinfo)) {
                 snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
                           "progress. Please retry after completion", volname);
@@ -2220,10 +2267,19 @@ glusterd_op_add_brick (dict_t *dict, char **op_errstr)
                         GD_MSG_BRICK_ADD_FAIL, "Unable to add bricks");
                 goto out;
         }
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                goto out;
+        if (priv->op_version <= GD_OP_VERSION_3_7_5) {
+               ret = glusterd_store_volinfo (volinfo,
+                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret)
+                        goto out;
+        } else {
+                 /*
+                 * The cluster is operating at version greater than
+                 * gluster-3.7.5. So no need to store volfiles
+                 * in commit phase, the same will be done
+                 * in post validate phase with v3 framework.
+                 */
+        }
 
         if (GLUSTERD_STATUS_STARTED == volinfo->status)
                 ret = glusterd_svcs_manager (volinfo);
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
index 2afa6966d1f..2814bb05430 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -179,6 +179,15 @@ gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
                         goto out;
                 }
                 break;
+        case GD_OP_ADD_BRICK:
+                ret = glusterd_op_stage_add_brick (dict, op_errstr, rsp_dict);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_WARNING, 0,
+                                GD_MSG_PRE_VALIDATION_FAIL,
+                                "Replace-brick prevalidation failed.");
+                        goto out;
+                }
+                break;
 
         default:
                 break;
@@ -264,6 +273,18 @@ gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
                         }
                         break;
                 }
+                case GD_OP_ADD_BRICK:
+                {
+                        ret = glusterd_op_add_brick (dict, op_errstr);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_COMMIT_OP_FAIL,
+                                        "Add-brick commit failed.");
+                                goto out;
+                        }
+                        break;
+
+                }
                default:
                        break;
         }
@@ -278,8 +299,11 @@ int32_t
 gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
                              char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
+        int32_t                  ret       = -1;
+        xlator_t                *this      = NULL;
+        char                    *volname   = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
+
 
         this = THIS;
         GF_ASSERT (this);
@@ -301,8 +325,36 @@ gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
                        }
                        break;
                }
+               case GD_OP_ADD_BRICK:
+               {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_msg ("glusterd", GF_LOG_ERROR, errno,
+                                        GD_MSG_DICT_GET_FAILED, "Unable to get"
+                                        " volume name");
+                                goto out;
+                        }
+
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret) {
+                                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
+                                        GD_MSG_VOL_NOT_FOUND, "Unable to "
+                                        "allocate memory");
+                                goto out;
+                        }
+                        ret = glusterd_create_volfiles_and_notify_services (
+                                                                     volinfo);
+                        if (ret)
+                                goto out;
+                        ret = glusterd_store_volinfo (volinfo,
+                                            GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                        if (ret)
+                                goto out;
+                        break;
+
+               }
                default:
-                       break;
+                        break;
         }
 
         ret = 0;
@@ -559,6 +611,15 @@ glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op,
                         goto out;
                 }
                 break;
+        case GD_OP_ADD_BRICK:
+                ret = glusterd_aggr_brick_mount_dirs (aggr, rsp);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL, "Failed to "
+                                "aggregate brick mount dirs");
+                        goto out;
+                }
+                break;
         default:
                 ret = -1;
                 gf_msg (this->name, GF_LOG_ERROR, EINVAL,
@@ -857,6 +918,7 @@ glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
                 case GD_OP_SNAP:
                         dict_copy (dict, req_dict);
                         break;
+                case GD_OP_ADD_BRICK:
                 case GD_OP_REPLACE_BRICK:
                 {
                         ret = dict_get_str (dict, "volname", &volname);
@@ -1881,9 +1943,8 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
         }
 
         /* POST-COMMIT VALIDATE PHASE */
-        /* As of now, post_validate is not handling any other
-           commands other than snapshot. So as of now, I am
-           sending 0 (op_ret as 0).
+        /* As of now, post_validate is not trying to cleanup any failed
+           commands. So as of now, I am sending 0 (op_ret as 0).
         */
         ret = glusterd_mgmt_v3_post_validate (op, 0, dict, req_dict, &op_errstr,
                                               txn_generation);