afr: heal metadata in discover code path

During graph switch, if fuse sends nameless (gfid) lookups, afr takes
the discover code path to serve it. If there are pending metadata heals,
they do not happen unless an inode refresh happens as a part of
discover (which is not guaranteed to happen always).

This patch fixes it by attempting metadata heal as a part of discover,
just like how it is done in lookup code path.

Also removed creating superfluous  heal frames when launching heal.

Change-Id: I49868649361ebe5d70b6ea150f4686169b6c3070
BUG: 1473636
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: https://review.gluster.org/17850
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Karthik U S <ksubrahm@redhat.com>

3 files changed, 60 insertions, 31 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index cba18b2ff8f..a255f9d14ad 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -45,6 +45,12 @@
 #include "afr-messages.h"
 #include "compound-fop-utils.h"
 
+typedef struct afr_lookup_heal_data {
+        call_frame_t *fop_frame;
+        void (*heal_done_cbk)(call_frame_t *, xlator_t *);
+        gf_boolean_t can_free;
+} afr_lookup_heal_data_t;
+
 int32_t
 afr_quorum_errno (afr_private_t *priv)
 {
@@ -1031,14 +1037,14 @@ afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode,
 	return ret;
 }
 
-
-
 int
 afr_refresh_selfheal_done (int ret, call_frame_t *heal, void *opaque)
 {
-	if (heal)
-		STACK_DESTROY (heal->root);
-	return 0;
+        afr_lookup_heal_data_t *data = opaque;
+
+        if (data && data->can_free)
+                GF_FREE (data);
+        return 0;
 }
 
 int
@@ -2393,7 +2399,8 @@ afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
 int
 afr_lookup_sh_metadata_wrap (void *opaque)
 {
-        call_frame_t *frame       = opaque;
+        afr_lookup_heal_data_t *data = opaque;
+        call_frame_t *frame       = data->fop_frame;
         afr_local_t  *local       = NULL;
         xlator_t     *this        = NULL;
         inode_t      *inode       = NULL;
@@ -2431,13 +2438,23 @@ afr_lookup_sh_metadata_wrap (void *opaque)
                               "Unable to set link-count in dict ");
         }
 
-        inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
-                                                 local->loc.name, local->replies,
-                                                 local->child_up, dict);
+        if (!local->loc.parent && gf_uuid_is_null (local->loc.pargfid)) {
+                ret = afr_selfheal_unlocked_discover_on (frame, local->inode,
+                                                         local->loc.gfid,
+                                                         local->replies,
+                                                         local->child_up);
+        } else {
+                inode = afr_selfheal_unlocked_lookup_on (frame,
+                                                         local->loc.parent,
+                                                         local->loc.name,
+                                                         local->replies,
+                                                         local->child_up, dict);
+        }
         if (inode)
                 inode_unref (inode);
 out:
-        afr_lookup_done (frame, this);
+        data->can_free = _gf_true;
+        data->heal_done_cbk (frame, this);
 
         if (dict)
                 dict_unref (dict);
@@ -2498,25 +2515,23 @@ afr_can_start_metadata_self_heal(call_frame_t *frame, xlator_t *this)
 }
 
 int
-afr_lookup_metadata_heal_check (call_frame_t *frame, xlator_t *this)
+afr_lookup_metadata_heal_check (afr_lookup_heal_data_t *data, xlator_t *this)
 
 {
-        call_frame_t *heal = NULL;
-        int ret            = 0;
+        call_frame_t *frame = data->fop_frame;
+        int ret            = -1;
 
         if (!afr_can_start_metadata_self_heal (frame, this))
                 goto out;
 
-        heal = copy_frame (frame);
-        if (heal)
-                heal->root->pid = GF_CLIENT_PID_SELF_HEALD;
         ret = synctask_new (this->ctx->env, afr_lookup_sh_metadata_wrap,
-                            afr_refresh_selfheal_done, heal, frame);
+                            afr_refresh_selfheal_done, NULL, data);
         if(ret)
                 goto out;
         return ret;
 out:
-        afr_lookup_done (frame, this);
+        data->can_free = _gf_true;
+        data->heal_done_cbk (frame, this);
         return ret;
 }
 
@@ -2524,7 +2539,8 @@ int
 afr_lookup_selfheal_wrap (void *opaque)
 {
         int ret = 0;
-	call_frame_t *frame = opaque;
+	afr_lookup_heal_data_t *data = opaque;
+	call_frame_t *frame = data->fop_frame;
 	afr_local_t *local = NULL;
 	xlator_t *this = NULL;
 	inode_t *inode = NULL;
@@ -2547,10 +2563,11 @@ afr_lookup_selfheal_wrap (void *opaque)
 	if (inode)
 		inode_unref (inode);
 
-        afr_lookup_metadata_heal_check(frame, this);
+        afr_lookup_metadata_heal_check (data, this);
         return 0;
 
 unwind:
+        data->can_free = _gf_true;
 	AFR_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
         return 0;
 }
@@ -2560,15 +2577,19 @@ afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)
 {
 	afr_local_t *local = NULL;
 	afr_private_t *priv = NULL;
-	call_frame_t *heal = NULL;
 	int i = 0, first = -1;
 	gf_boolean_t need_heal = _gf_false;
 	struct afr_reply *replies = NULL;
+        afr_lookup_heal_data_t *data = NULL;
 	int ret = 0;
 
 	local = frame->local;
 	replies = local->replies;
 	priv = this->private;
+        data = GF_CALLOC (1, sizeof (afr_lookup_heal_data_t),
+                          gf_afr_mt_afr_lookup_heal_data_t);
+        data->fop_frame = frame;
+        data->heal_done_cbk = afr_lookup_done;
 
 	for (i = 0; i < priv->child_count; i++) {
 		if (!replies[i].valid)
@@ -2596,18 +2617,16 @@ afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)
 	}
 
 	if (need_heal) {
-
-		heal = copy_frame (frame);
-		if (heal)
-			heal->root->pid = GF_CLIENT_PID_SELF_HEALD;
 		ret = synctask_new (this->ctx->env, afr_lookup_selfheal_wrap,
-				    afr_refresh_selfheal_done, heal, frame);
+				    afr_refresh_selfheal_done, NULL, data);
 		if (ret)
 			goto metadata_heal;
                 return ret;
 	}
 metadata_heal:
-        ret = afr_lookup_metadata_heal_check (frame, this);
+        ret = afr_lookup_metadata_heal_check (data, this);
+        if (ret && data->can_free)
+                GF_FREE (data);
 
 	return ret;
 }
@@ -2661,8 +2680,6 @@ afr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	return 0;
 }
 
-
-
 static void
 afr_discover_done (call_frame_t *frame, xlator_t *this)
 {
@@ -2738,6 +2755,7 @@ afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int             child_index     = -1;
         GF_UNUSED int ret               = 0;
 	int8_t need_heal                = 1;
+        afr_lookup_heal_data_t *data    = NULL;
 
 	child_index = (long) cookie;
 
@@ -2765,8 +2783,14 @@ afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         call_count = afr_frame_return (frame);
         if (call_count == 0) {
-               afr_set_need_heal (this, local);
-               afr_discover_done (frame, this);
+                data = GF_CALLOC (1, sizeof (afr_lookup_heal_data_t),
+                                  gf_afr_mt_afr_lookup_heal_data_t);
+                data->fop_frame = frame;
+                data->heal_done_cbk = afr_discover_done;
+                afr_set_need_heal (this, local);
+                ret = afr_lookup_metadata_heal_check (data, this);
+                if (ret && data->can_free)
+                        GF_FREE (data);
         }
 
 	return 0;
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index c7d6261b110..a25c7b4f07b 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -47,6 +47,7 @@ enum gf_afr_mem_types_ {
         gf_afr_mt_spb_status_t,
         gf_afr_mt_empty_brick_t,
         gf_afr_mt_child_latency_t,
+        gf_afr_mt_afr_lookup_heal_data_t,
     gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index 36f081ec354..82608d261d5 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -152,6 +152,10 @@ afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
 int
 afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
 				uuid_t gfid, struct afr_reply *replies);
+int
+afr_selfheal_unlocked_discover_on (call_frame_t *frame, inode_t *inode,
+				   uuid_t gfid, struct afr_reply *replies,
+                                   unsigned char *discover_on);
 
 inode_t *
 afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,