glusterd / afr : Enable auto heal when replica count increases

In replicate volumes, when a brick is added to a replicate
group, heal to the new brick should be triggered.
Also, the new brick should not be considered as source for
healing till it is up to date.

Previously, extended attributes had to be set manually on
the bricks for this to happen. This patch is part 1 patch
to automate this process.

Change-Id: I29958448618372bfde23bf1dac5dd23dba1ad98f
BUG: 1276203
Signed-off-by: Anuradha Talur <atalur@redhat.com>
Reviewed-on: http://review.gluster.org/12451
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Ravishankar N <ravishankar@redhat.com>
Smoke: Gluster Build System <jenkins@build.gluster.com>

8 files changed, 280 insertions, 98 deletions

diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 9bc12f74eaa..cf9ae9a417b 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -62,11 +62,9 @@
 
 #define GD_OP_VERSION_3_7_7    30707 /* Op-version for GlusterFS 3.7.7 */
 
-/* Op-version was not bumped up for 3.7.8 */
-
 #define GD_OP_VERSION_3_7_9    30709 /* Op-version for GlusterFS 3.7.9 */
 
-#define GD_OP_VERSION_3_7_10    30710 /* Op-version for GlusterFS 3.7.10 */
+#define GD_OP_VERSION_3_7_10   30710 /* Op-version for GlusterFS 3.7.10 */
 
 #define GD_OP_VERSION_4_0_0    40000 /* Op-version for GlusterFS 4.0.0 */
 
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 9dcaadfa645..a037cb5cbbd 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -171,6 +171,7 @@
 #define GF_AFR_SBRAIN_CHOICE "replica.split-brain-choice"
 #define GF_AFR_SPB_CHOICE_TIMEOUT "replica.split-brain-choice-timeout"
 #define GF_AFR_SBRAIN_RESOLVE "replica.split-brain-heal-finalize"
+#define GF_AFR_ADD_BRICK "trusted.add-brick"
 #define GF_AFR_REPLACE_BRICK "trusted.replace-brick"
 #define GF_AFR_DIRTY "trusted.afr.dirty"
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index d77626d3457..920f7f05623 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -21,6 +21,7 @@
 #include "glusterd-messages.h"
 #include "glusterd-server-quorum.h"
 #include "run.h"
+#include "glusterd-volgen.h"
 #include <sys/signal.h>
 
 /* misc */
@@ -1233,6 +1234,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         char                         *brick_mount_dir  = NULL;
         xlator_t                     *this           = NULL;
         glusterd_conf_t              *conf           = NULL;
+        gf_boolean_t                  is_valid_add_brick = _gf_false;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1320,6 +1322,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         /* Gets changed only if the options are given in add-brick cli */
         if (type)
                 volinfo->type = type;
+
         if (replica_count) {
                 volinfo->replica_count = replica_count;
         }
@@ -1355,6 +1358,27 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                         CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
 #endif
 
+        /* This check needs to be added to distinguish between
+         * attach-tier commands and add-brick commands.
+         * When a tier is attached, adding is done via add-brick
+         * and setting of pending xattrs shouldn't be done for
+         * attach-tiers as they are virtually new volumes.
+         */
+        if (glusterd_is_volume_replicate (volinfo)) {
+                if (replica_count &&
+                    !dict_get (dict, "attach-tier") &&
+                    conf->op_version >= GD_OP_VERSION_3_7_10) {
+                        is_valid_add_brick = _gf_true;
+                        ret = generate_dummy_client_volfiles (volinfo);
+                        if (ret) {
+                                gf_msg (THIS->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_VOLFILE_CREATE_FAIL,
+                                        "Failed to create volfile.");
+                                goto out;
+                                }
+                        }
+        }
+
         while (i <= count) {
                 ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
                                                               &brickinfo);
@@ -1386,6 +1410,16 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                         }
                 }
 
+                /* if the volume is a replicate volume, do: */
+                if (is_valid_add_brick) {
+                        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
+                                ret = glusterd_handle_replicate_brick_ops (
+                                                           volinfo, brickinfo,
+                                                           GD_OP_ADD_BRICK);
+                                if (ret < 0)
+                                        goto out;
+                        }
+                }
                 ret = glusterd_brick_start (volinfo, brickinfo,
                                             _gf_true);
                 if (ret)
@@ -1514,22 +1548,6 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         conf = this->private;
         GF_ASSERT (conf);
 
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (ret) {
-                gf_msg_debug (THIS->name, 0,
-                        "Unable to get replica count");
-        }
-
-        if (replica_count > 0) {
-                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
-                                        msg, sizeof(msg));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OP_VERSION_MISMATCH, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-        }
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
                 gf_msg (THIS->name, GF_LOG_ERROR, errno,
@@ -1550,6 +1568,42 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         if (ret)
                 goto out;
 
+        ret = dict_get_int32 (dict, "replica-count", &replica_count);
+        if (ret) {
+                gf_msg_debug (THIS->name, 0,
+                        "Unable to get replica count");
+        }
+
+        if (replica_count > 0) {
+                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
+                                        msg, sizeof(msg));
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_OP_VERSION_MISMATCH, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
+                        goto out;
+                }
+        }
+
+        /* Do not allow add-brick for stopped volumes when replica-count
+         * is being increased.
+         */
+        if (glusterd_is_volume_replicate (volinfo)) {
+                if (conf->op_version >= GD_OP_VERSION_3_7_10 &&
+                    !dict_get (dict, "attach-tier") &&
+                    replica_count &&
+                    GLUSTERD_STATUS_STOPPED == volinfo->status) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg), " Volume must not be in"
+                                  " stopped state when replica-count needs to "
+                                  " be increased.");
+                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
+                                GD_MSG_BRICK_ADD_FAIL, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
+                        goto out;
+                }
+        }
+
         if (conf->op_version > GD_OP_VERSION_3_7_5 &&
             is_origin_glusterd (dict)) {
                 ret = glusterd_validate_quorum (this, GD_OP_ADD_BRICK, dict,
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 093548de712..535f1256baf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -39,83 +39,6 @@ glusterd_mgmt_v3_initiate_replace_brick_cmd_phases (rpcsvc_request_t *req,
                                                     glusterd_op_t op,
                                                     dict_t *dict);
 int
-glusterd_handle_replicate_replace_brick (glusterd_volinfo_t *volinfo,
-                                         glusterd_brickinfo_t *brickinfo)
-{
-        int32_t                    ret               = -1;
-        char                       tmpmount[]        = "/tmp/mntXXXXXX";
-        char                       logfile[PATH_MAX] = {0,};
-        int                        dirty[3]          = {0,};
-        runner_t                   runner            = {0};
-        glusterd_conf_t           *priv              = NULL;
-        char                      *pid               = NULL;
-        char                      *volfileserver     = NULL;
-
-        priv = THIS->private;
-
-        dirty[2] = hton32(1);
-
-        ret = sys_lsetxattr (brickinfo->path, GF_AFR_DIRTY, dirty,
-                             sizeof (dirty), 0);
-        if (ret == -1) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"
-                        " attribute %s : %s.", GF_AFR_DIRTY, strerror (errno));
-                goto out;
-        }
-
-        if (mkdtemp (tmpmount) == NULL) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "failed to create a temporary mount directory.");
-                ret = -1;
-                goto out;
-        }
-        snprintf (logfile, sizeof (logfile),
-                  DEFAULT_LOG_FILE_DIRECTORY"/%s-replace-brick-mount.log",
-                  volinfo->volname);
-
-        ret = gf_asprintf (&pid, "%d", GF_CLIENT_PID_SELF_HEALD);
-        if (ret < 0)
-                goto out;
-
-        if (dict_get_str (THIS->options, "transport.socket.bind-address",
-                          &volfileserver) != 0)
-                volfileserver = "localhost";
-
-        runinit (&runner);
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", volfileserver,
-                         "--volfile-id", volinfo->volname,
-                         "--client-pid", pid,
-                         "-l", logfile, tmpmount, NULL);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-
-        if (ret) {
-                runner_log (&runner, THIS->name, GF_LOG_ERROR, "mount command"
-                            "failed.");
-                goto lock;
-        }
-        ret = sys_lsetxattr (tmpmount, GF_AFR_REPLACE_BRICK,
-                             brickinfo->brick_id, sizeof (brickinfo->brick_id),
-                             0);
-        if (ret == -1)
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"
-                        " attribute %s : %s", GF_AFR_REPLACE_BRICK,
-                        strerror (errno));
-        gf_umount_lazy (THIS->name, tmpmount, 1);
-lock:
-        synclock_lock (&priv->big_lock);
-out:
-        if (pid)
-                GF_FREE (pid);
-        gf_msg_debug ("glusterd", 0, "Returning with ret");
-        return ret;
-}
-
-int
 __glusterd_handle_replace_brick (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
@@ -670,8 +593,8 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
         /* if the volume is a replicate volume, do: */
         if (glusterd_is_volume_replicate (volinfo)) {
                 if (!gf_uuid_compare (new_brickinfo->uuid, MY_UUID)) {
-                        ret = glusterd_handle_replicate_replace_brick
-                                  (volinfo, new_brickinfo);
+                        ret = glusterd_handle_replicate_brick_ops (volinfo,
+                                        new_brickinfo, GD_OP_REPLACE_BRICK);
                         if (ret < 0)
                                 goto out;
                 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 50cc4476fad..a0cc5d409c8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7100,6 +7100,33 @@ void glusterd_update_tier_status (glusterd_volinfo_t *volinfo) {
 }
 
 int
+glusterd_get_dummy_client_filepath (char *filepath,
+                                    glusterd_volinfo_t *volinfo,
+                                    gf_transport_type type)
+{
+        int   ret             = 0;
+        char  path[PATH_MAX]  = {0,};
+
+        switch (type) {
+        case GF_TRANSPORT_TCP:
+        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                snprintf (filepath, PATH_MAX,
+                          "/tmp/%s.tcp-fuse.vol", volinfo->volname);
+                break;
+
+        case GF_TRANSPORT_RDMA:
+                snprintf (filepath, PATH_MAX,
+                          "/tmp/%s.rdma-fuse.vol", volinfo->volname);
+                break;
+        default:
+                ret = -1;
+                break;
+        }
+
+        return ret;
+}
+
+int
 glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
                               size_t len, int cmd, defrag_cbk_fn_t cbk)
 {
@@ -11074,3 +11101,117 @@ gd_get_shd_key (int type)
         }
         return key;
 }
+
+int
+glusterd_handle_replicate_brick_ops (glusterd_volinfo_t *volinfo,
+                                     glusterd_brickinfo_t *brickinfo,
+                                     glusterd_op_t op)
+{
+        int32_t                    ret               = -1;
+        char                       tmpmount[]        = "/tmp/mntXXXXXX";
+        char                       logfile[PATH_MAX] = {0,};
+        int                        dirty[3]          = {0,};
+        runner_t                   runner            = {0};
+        glusterd_conf_t           *priv              = NULL;
+        char                      *pid               = NULL;
+        char                       vpath[PATH_MAX]   = {0,};
+        char                      *volfileserver     = NULL;
+
+        priv = THIS->private;
+        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
+
+        dirty[2] = hton32(1);
+
+        ret = sys_lsetxattr (brickinfo->path, GF_AFR_DIRTY, dirty,
+                             sizeof (dirty), 0);
+        if (ret == -1) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"
+                        " attribute %s : %s.", GF_AFR_DIRTY, strerror (errno));
+                goto out;
+        }
+
+        if (mkdtemp (tmpmount) == NULL) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        GD_MSG_DIR_OP_FAILED,
+                        "failed to create a temporary mount directory.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = gf_asprintf (&pid, "%d", GF_CLIENT_PID_SELF_HEALD);
+        if (ret < 0)
+                goto out;
+
+        switch (op) {
+        case GD_OP_REPLACE_BRICK:
+        if (dict_get_str (THIS->options, "transport.socket.bind-address",
+                          &volfileserver) != 0)
+                volfileserver = "localhost";
+
+                snprintf (logfile, sizeof (logfile),
+                          DEFAULT_LOG_FILE_DIRECTORY"/%s-replace-brick-mount.log",
+                          volinfo->volname);
+                if (!*logfile) {
+                        ret = -1;
+                        goto out;
+                }
+                runinit (&runner);
+                runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                                 "-s", volfileserver,
+                                 "--volfile-id", volinfo->volname,
+                                 "--client-pid", pid,
+                                 "-l", logfile, tmpmount, NULL);
+                break;
+
+        case GD_OP_ADD_BRICK:
+                snprintf (logfile, sizeof (logfile),
+                          DEFAULT_LOG_FILE_DIRECTORY"/%s-add-brick-mount.log",
+                          volinfo->volname);
+                if (!*logfile) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_get_dummy_client_filepath (vpath, volinfo,
+                                                    volinfo->transport_type);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get "
+                                "volfile path");
+                        goto out;
+                }
+                runinit (&runner);
+                runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                                 "--volfile", vpath,
+                                 "--client-pid", pid,
+                                 "-l", logfile, tmpmount, NULL);
+                break;
+        default:
+                break;
+        }
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "mount command"
+                        " failed.");
+                goto lock;
+        }
+        ret = sys_lsetxattr (tmpmount, (op == GD_OP_REPLACE_BRICK) ?
+                             GF_AFR_REPLACE_BRICK : GF_AFR_ADD_BRICK,
+                             brickinfo->brick_id, sizeof (brickinfo->brick_id),
+                             0);
+        if (ret == -1)
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"
+                        " attribute %s : %s", (op == GD_OP_REPLACE_BRICK) ?
+                        GF_AFR_REPLACE_BRICK : GF_AFR_ADD_BRICK,
+                        strerror (errno));
+        gf_umount_lazy (THIS->name, tmpmount, 1);
+lock:
+        synclock_lock (&priv->big_lock);
+out:
+        if (pid)
+                GF_FREE (pid);
+        gf_msg_debug ("glusterd", 0, "Returning with ret");
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 7195fe30658..50d691acad5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -690,4 +690,14 @@ int
 glusterd_volume_brick_for_each (glusterd_volinfo_t *volinfo, void *data,
                int (*fn) (glusterd_volinfo_t *, glusterd_brickinfo_t *,
                           dict_t *mod_dict, void *));
+
+int
+glusterd_get_dummy_client_filepath (char *filepath,
+                                    glusterd_volinfo_t *volinfo,
+                                    gf_transport_type type);
+
+int
+glusterd_handle_replicate_brick_ops (glusterd_volinfo_t *volinfo,
+                                     glusterd_brickinfo_t *brickinfo,
+                                     glusterd_op_t op);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index cc599846bb9..b61c056507b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -5451,6 +5451,58 @@ enumerate_transport_reqs (gf_transport_type type, char **types)
 }
 
 int
+generate_dummy_client_volfiles (glusterd_volinfo_t *volinfo)
+{
+        int                i                  = 0;
+        int                ret                = -1;
+        char               filepath[PATH_MAX] = {0,};
+        char               *types[]           = {NULL, NULL, NULL};
+        dict_t             *dict              = NULL;
+        xlator_t           *this              = NULL;
+        gf_transport_type  type               = GF_TRANSPORT_TCP;
+
+        this = THIS;
+
+        enumerate_transport_reqs (volinfo->transport_type, types);
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+        for (i = 0; types[i]; i++) {
+                memset (filepath, 0, sizeof (filepath));
+                ret = dict_set_str (dict, "client-transport-type", types[i]);
+                if (ret)
+                        goto out;
+                type = transport_str_to_type (types[i]);
+
+                ret = dict_set_uint32 (dict, "trusted-client", GF_CLIENT_OTHER);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_get_dummy_client_filepath (filepath,
+                                                          volinfo, type);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
+                                GD_MSG_INVALID_ENTRY,
+                                "Received invalid transport-type.");
+                        goto out;
+                }
+
+                ret = generate_single_transport_client_volfile (volinfo,
+                                                                filepath,
+                                                                dict);
+                if (ret)
+                        goto out;
+        }
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        gf_msg_trace ("glusterd", 0, "Returning %d", ret);
+        return ret;
+}
+
+int
 generate_client_volfiles (glusterd_volinfo_t *volinfo,
                           glusterd_client_type_t client_type)
 {
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 36a50919d3c..9d206f692c1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -288,4 +288,7 @@ glusterd_volopt_validate (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
 gf_boolean_t
 gd_is_self_heal_enabled (glusterd_volinfo_t *volinfo, dict_t *dict);
 
+int
+generate_dummy_client_volfiles (glusterd_volinfo_t *volinfo);
+
 #endif