afr: add mtime based split-brain resolution to CLI

Extended the CLI to include support for split-brain resolution based on
mtime. The command syntax is:

$:gluster volume heal <VOLNAME> split-brain latest-mtime <FILE>

where <FILE> can be either the full file name as seen from the root of the
volume (or) the gfid-string representation of the file.

Change-Id: I7a16f72ff1a4495aa69f43f22758a9404e958b4f
BUG: 1321322
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: http://review.gluster.org/13828
Smoke: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

2 files changed, 72 insertions, 10 deletions

diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index f12ce81b0ee..151960d2516 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -328,6 +328,10 @@ afr_mark_largest_file_as_source (xlator_t *this, unsigned char *sources,
         for (i = 0; i < priv->child_count; i++) {
                 if (!sources[i])
                         continue;
+                if (!replies[i].valid || replies[i].op_ret != 0) {
+                        sources[i] = 0;
+                        continue;
+                }
                 if (size <= replies[i].poststat.ia_size) {
                         size = replies[i].poststat.ia_size;
                 }
@@ -343,6 +347,41 @@ afr_mark_largest_file_as_source (xlator_t *this, unsigned char *sources,
 }
 
 void
+afr_mark_latest_mtime_file_as_source (xlator_t *this, unsigned char *sources,
+                                 struct afr_reply *replies)
+{
+        int i = 0;
+        afr_private_t *priv = NULL;
+        uint32_t mtime = 0;
+        uint32_t mtime_nsec = 0;
+
+        priv = this->private;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!sources[i])
+                        continue;
+                if (!replies[i].valid || replies[i].op_ret != 0) {
+                        sources[i] = 0;
+                        continue;
+                }
+                if ((mtime < replies[i].poststat.ia_mtime) ||
+                    ((mtime == replies[i].poststat.ia_mtime) &&
+                     (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+                        mtime = replies[i].poststat.ia_mtime;
+                        mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                }
+        }
+        for (i = 0; i < priv->child_count; i++) {
+                if (!sources[i])
+                        continue;
+                if ((mtime > replies[i].poststat.ia_mtime) ||
+                    ((mtime == replies[i].poststat.ia_mtime) &&
+                     (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
+                        sources[i] = 0;
+                }
+        }
+}
+
+void
 afr_mark_active_sinks (xlator_t *this, unsigned char *sources,
                        unsigned char *locked_on, unsigned char *sinks)
 {
@@ -432,6 +471,9 @@ afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
         }
         xdata_rsp = local->xdata_rsp;
 
+        for (i = 0 ; i < priv->child_count; i++)
+                if (locked_on[i])
+                        sources[i] = 1;
         switch (heal_op) {
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
                 if (type == AFR_METADATA_TRANSACTION) {
@@ -442,9 +484,6 @@ afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
                                 ret = -1;
                         goto out;
                 }
-                for (i = 0 ; i < priv->child_count; i++)
-                        if (locked_on[i])
-                                sources[i] = 1;
                 afr_mark_largest_file_as_source (this, sources, replies);
                 if (AFR_COUNT (sources, priv->child_count) != 1) {
                         ret = dict_set_str (xdata_rsp, "sh-fail-msg",
@@ -453,11 +492,24 @@ afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
                                 ret = -1;
                         goto out;
                 }
-                for (i = 0 ; i < priv->child_count; i++)
-                        if (sources[i])
-                                source = i;
-                sinks[source] = 0;
-                healed_sinks[source] = 0;
+                break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+                if (type == AFR_METADATA_TRANSACTION) {
+                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
+                                            "Use source-brick option to"
+                                            " heal metadata split-brain");
+                        if (!ret)
+                                ret = -1;
+                        goto out;
+                }
+                afr_mark_latest_mtime_file_as_source (this, sources, replies);
+                if (AFR_COUNT (sources, priv->child_count) != 1) {
+                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
+                                            "No difference in mtime");
+                        if (!ret)
+                                ret = -1;
+                        goto out;
+                }
                 break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
                 ret = dict_get_str (xdata_req, "child-name", &name);
@@ -478,16 +530,25 @@ afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
                                 ret = -1;
                         goto out;
                 }
+                memset (sources, 0, sizeof (*sources) * priv->child_count);
                 sources[source] = 1;
-                sinks[source] = 0;
-                healed_sinks[source] = 0;
                 break;
         default:
                 ret = -1;
                 goto out;
         }
+        for (i = 0 ; i < priv->child_count; i++) {
+                if (sources[i]) {
+                        source = i;
+                        break;
+                }
+        }
+        sinks[source] = 0;
+        healed_sinks[source] = 0;
         ret = source;
 out:
+        if (ret < 0)
+                memset (sources, 0, sizeof (*sources) * priv->child_count);
         return ret;
 
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 567b1acc58a..ad4bae455e7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1818,6 +1818,7 @@ glusterd_handle_heal_cmd (xlator_t *this, glusterd_volinfo_t *volinfo,
         case GF_SHD_OP_HEAL_ENABLE: /* This op should be handled in volume-set*/
         case GF_SHD_OP_HEAL_DISABLE:/* This op should be handled in volume-set*/
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:/*glfsheal cmd*/
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:/*glfsheal cmd*/
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:/*glfsheal cmd*/
                 ret = -1;
                 *op_errstr = gf_strdup("Invalid heal-op");