afr/cluster: PGFID heal support

Summary:
  PGFID healing enables heals which might otherwise fail due
  due to the lack of a entry heal to succeed by performing
  the entry healing within the same heal flow.

  It does this by leveraging the PGFID tracking feature of
  the POSIX xlator, and examining lookup replies for the
  PGFID attribute.  If detected, the pgfid will be decoded
  and stored for later use in case the heal fails for whatever
  reason.  Cascading heal failures are handled through
  recursion.

  This feature is critical for a couple reasons:
      1. General healing predictability - When the SHD
         attempts to heal a given GFID, it should be able
         to do so without having to wait for some other
         dependent heal to take place.
      2. Reliability - In some cases the parent directory
         may require healing, but the req'd entry in the
         indices/xattrop directory may not exist
         (e.g. bugs/crashes etc).  Prior to PGFID heal support
         some sort of external script would be required to
         queue up these heals by using FS specific utilities
         to lookup the parent directory by hardlink or
         worse...do a costly full heal to clean them up.
      3. Performance - In combination with multi-threaded SHD
         this feature will make SHD healing _much_ faster as
         directories with large amount of files to be healed
         will no longer have to wait for an entry heal to
         come along, the first file in that directory queued
         for healing will trigger an entry heal for the directory
         and this will allow the other files in that directory
         to be (immediatelly) healed in parallel.

Test Plan:
- run prove tests/basic/afr/shd_pgfid_heal.t
- run prove tests/basic/afr/shd*.t
- run prove tests/basic/afr/gfid*.t

Differential Revision: https://phabricator.fb.com/D2546133

Change-Id: I25f586047f8bcafa900c0cc9ee8f0e2128688c73
Signed-off-by: Jeff Darcy <jdarcy@fb.com>
Reviewed-on: https://review.gluster.org/17929
Smoke: Gluster Build System <jenkins@build.gluster.org>
Tested-by: Jeff Darcy <jeff@pl.atyp.us>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

6 files changed, 127 insertions, 17 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index cfea53208c8..333ea888395 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1381,6 +1381,12 @@ afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req)
                               "Unable to set list-xattr in dict ");
         }
 
+        ret = dict_set_int32 (xattr_req, GET_ANCESTRY_PATH_KEY, 42);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to set ancestry path key in dict ");
+        }
+
 	return ret;
 }
 
@@ -1751,6 +1757,8 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         GF_FREE (local->readable);
         GF_FREE (local->readable2);
 
+        GF_FREE (local->heal_ancestry_path);
+
 	if (local->inode)
 		inode_unref (local->inode);
 
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 12b031d78e6..98ad65f29fd 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1212,7 +1212,6 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-
         /* count the number of dirty fops witnessed */
         for (i = 0; i < priv->child_count; i++)
                 witness[i] += dirty[i];
@@ -1220,6 +1219,67 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
 	return 0;
 }
 
+/*
+ * This function will examine a reply and look for a PGFID xattr
+ * and if found will record this in the frame's local struct.
+ *
+ * This can then be used to fall-back to healing the parent
+ * directory in cases where metadata/data healing isn't yet
+ * possible because an entry heal of the parent directory has not
+ * yet taken place.
+ *
+ * This is critical for a couple reasons:
+ *      1. General healing predictability - When the SHD
+ *         attempts to heal a given GFID, it should be able
+ *         to do so without having to wait for some other
+ *         dependent heal to take place.
+ *      2. Reliability - In some cases the parent directory
+ *         may require healing, but the req'd entry in the
+ *         indices/xattrop directory may not exist
+ *         (e.g. bugs/crashes etc).  This feature removes
+ *
+ */
+void
+_afr_set_heal_pgfid_from_reply (xlator_t *this, afr_local_t  *local,
+                                struct afr_reply reply)
+{
+        data_pair_t *trav = reply.xdata->members_list;
+        uuid_t *pgfid = NULL;
+        int32_t ret = 0;
+        int32_t pgfid_prefix_len = sizeof (PGFID_XATTR_KEY_PREFIX) - 1;
+        char *pgfid_str = NULL;
+        data_t *ancestry_path_data = NULL;
+        char *ancestry_path = "Unknown";
+
+        pgfid = &local->heal_pgfid;
+
+        while (trav) {
+                if (!strncmp (PGFID_XATTR_KEY_PREFIX, trav->key,
+                    pgfid_prefix_len)) {
+                        pgfid_str = trav->key + pgfid_prefix_len;
+                        ret = gf_uuid_parse (pgfid_str, *pgfid);
+                        break;
+                }
+                trav = trav->next;
+        }
+
+        if (!ret && !gf_uuid_is_null (*pgfid)) {
+                if (!dict_lookup (reply.xdata,
+                                "glusterfs.ancestry.path",
+                                &ancestry_path_data)) {
+                        ancestry_path = data_to_str (
+                            ancestry_path_data);
+                        /* Allocation free'd on local destroy */
+                        local->heal_ancestry_path =
+                                gf_strdup (ancestry_path);
+                }
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Found pgfid (%s) for %s",
+                        uuid_utoa (*pgfid),
+                        ancestry_path);
+        }
+}
+
 void
 afr_log_selfheal (uuid_t gfid, xlator_t *this, int ret, char *type,
                   int source, unsigned char *sources,
@@ -1788,6 +1848,8 @@ afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
 			       gf_boolean_t *entry_selfheal)
 {
 	afr_private_t *priv = NULL;
+        afr_local_t  *local = NULL;
+
         inode_t *inode = NULL;
 	int i = 0;
 	int valid_cnt = 0;
@@ -1796,6 +1858,7 @@ afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
 	int ret = -1;
 
 	priv = this->private;
+        local = frame->local;
 
         inode = afr_inode_find (this, gfid);
         if (!inode)
@@ -1813,6 +1876,10 @@ afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
 		if (replies[i].op_ret == -1)
 			continue;
 
+                if (gf_uuid_is_null(local->heal_pgfid))
+                        _afr_set_heal_pgfid_from_reply (this,
+                            frame->local, replies[i]);
+
                 /* The data segment of the changelog can be non-zero to indicate
                  * the directory needs a full heal. So the check below ensures
                  * it's not a directory before setting the data_selfheal boolean.
@@ -2073,6 +2140,7 @@ afr_selfheal_do (call_frame_t *frame, xlator_t *this, uuid_t gfid)
 					     &data_selfheal,
 					     &metadata_selfheal,
 					     &entry_selfheal);
+
 	if (ret)
 		goto out;
 
@@ -2119,10 +2187,16 @@ int
 afr_selfheal (xlator_t *this, uuid_t gfid)
 {
         int           ret   = -1;
-	call_frame_t *frame = NULL;
-        afr_local_t *local = NULL;
+        gf_boolean_t  tried_parent = _gf_false;
+ 	call_frame_t *frame = NULL;
+        afr_local_t  *local = NULL;
+        char *ancestry_path = "Unknown";
+        char *pgfid_str = NULL;
+        char *gfid_str = NULL;
+ 
+heal_gfid:
+ 	frame = afr_frame_create (this);
 
-	frame = afr_frame_create (this);
 	if (!frame)
 		return ret;
 
@@ -2131,6 +2205,42 @@ afr_selfheal (xlator_t *this, uuid_t gfid)
 
         ret = afr_selfheal_do (frame, this, gfid);
 
+        if (tried_parent == _gf_false && ret &&
+                        !gf_uuid_is_null (local->heal_pgfid)) {
+                tried_parent = _gf_true;
+                pgfid_str = alloca (strlen (UUID0_STR) + 1);
+                gfid_str = alloca (strlen (UUID0_STR) + 1);
+                uuid_utoa_r (local->heal_pgfid, pgfid_str);
+                uuid_utoa_r (gfid, gfid_str);
+                if (local->heal_ancestry_path)
+                        ancestry_path = local->heal_ancestry_path;
+                gf_log (this->name, GF_LOG_INFO,
+                        "PGFID Healing - Heal failed for %s (%s), "
+                        "but found parent gfid (%s), attempting to heal "
+                        "parent directory by gfid.",
+                        gfid_str,
+                        ancestry_path,
+                        pgfid_str);
+                ret = afr_selfheal (this, local->heal_pgfid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "PGFID Healing - Healing of parent gfid "
+                                "(%s) unsuccessful!  Healing of %s (%s) "
+                                "failed.",
+                                pgfid_str,
+                                gfid_str,
+                                ancestry_path);
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "PGFID Healing - Healing of parent gfid %s "
+                                "successful!  Re-attempting heal of %s (%s).",
+                                pgfid_str,
+                                gfid_str,
+                                ancestry_path);
+                        goto heal_gfid;
+                }
+        }
+
 	if (frame)
 		AFR_STACK_DESTROY (frame);
 
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 13e82f9aad4..78c9f0a3ab9 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -688,7 +688,6 @@ afr_selfheal_entry_do_subvol (call_frame_t *frame, xlator_t *this,
 	xlator_t *subvol = NULL;
 	afr_private_t *priv = NULL;
         gf_boolean_t mismatch = _gf_false;
-        afr_local_t *iter_local = NULL;
         afr_local_t *local = NULL;
         loc_t loc = {0,};
 
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index aa2af38e8cf..abd2f470131 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -825,6 +825,8 @@ typedef struct _afr_local {
         gf_boolean_t need_full_crawl;
         gf_boolean_t is_read_txn;
         loc_t           *unsplit_locs;   /* Un-split targets */
+        uuid_t          heal_pgfid;  /* pgfid of file being healed */
+        char            *heal_ancestry_path;  /* Full path if avail */
 } afr_local_t;
 
 
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index be98f2a1cca..9201f38f7ff 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -390,13 +390,6 @@ _is_quota_internal_xattr (dict_t *d, char *k, data_t *v, void *data)
         if (fnmatch ("trusted.glusterfs.quota*", k, 0) == 0)
                 return _gf_true;
 
-        /* It would be nice if posix filters pgfid xattrs. But since marker
-         * also takes up responsibility to clean these up, adding the filtering
-         * here (Check 'quota_xattr_cleaner')
-         */
-        if (fnmatch (PGFID_XATTR_KEY_PREFIX"*", k, 0) == 0)
-                return _gf_true;
-
         return _gf_false;
 }
 
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 3cbb947d6b2..eb0986ffb6a 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -212,7 +212,7 @@ posix_lookup (call_frame_t *frame, xlator_t *this,
         }
 
         if (priv->update_pgfid_nlinks) {
-                if (!gf_uuid_is_null (loc->pargfid) && !IA_ISDIR (buf.ia_type)) {
+                if (!gf_uuid_is_null (loc->pargfid)) {
                         MAKE_PGFID_XATTR_KEY (pgfid_xattr_key,
                                               PGFID_XATTR_KEY_PREFIX,
                                               loc->pargfid);
@@ -2543,8 +2543,7 @@ posix_rename (call_frame_t *frame, xlator_t *this,
 
         LOCK (&oldloc->inode->lock);
         {
-                if (!IA_ISDIR (oldloc->inode->ia_type)
-                    && priv->update_pgfid_nlinks) {
+                if (priv->update_pgfid_nlinks) {
                         MAKE_PGFID_XATTR_KEY (pgfid_xattr_key,
                                               PGFID_XATTR_KEY_PREFIX,
                                               oldloc->pargfid);
@@ -2610,8 +2609,7 @@ posix_rename (call_frame_t *frame, xlator_t *this,
                                 P_MSG_SET_XDATA_FAIL, "failed to set "
                                 GET_LINK_COUNT" for %s", real_newpath);
 
-                if (!IA_ISDIR (oldloc->inode->ia_type)
-                    && priv->update_pgfid_nlinks) {
+                if (priv->update_pgfid_nlinks) {
                         MAKE_PGFID_XATTR_KEY (pgfid_xattr_key,
                                               PGFID_XATTR_KEY_PREFIX,
                                               newloc->pargfid);