afr, index: Clean up stale directory and file indices in granular entry sh

	Backport of: http://review.gluster.org/14832

Specifically when a directory tree is removed (rm -rf)
while a brick is down, both the directory index and the
name indices of the files and subdirs under it will remain.
Self-heal will need to pick up these and remove them.

Towards this, afr sh will now also crawl indices/entry-changes
and call an rmdir on the dir if the directory index is stale.

On the brick side, rmdir fop has been implemented for index xl,
which would delete the directory index and its contents if present
in a synctask.

Change-Id: I08f45201adca56737ec2be1aab5433aebaefefd0
BUG: 1355609
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: http://review.gluster.org/14920
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

8 files changed, 243 insertions, 40 deletions

diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 45886dae3ee..09dfd450102 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -4426,6 +4426,21 @@ fop_enum_to_string (glusterfs_fop_t fop)
         return "UNKNOWNFOP";
 }
 
+const char *
+gf_inode_type_to_str (ia_type_t type)
+{
+        static const char *const str_ia_type[] = {
+                "UNKNOWN",
+                "REGULAR FILE",
+                "DIRECTORY",
+                "LINK",
+                "BLOCK DEVICE",
+                "CHARACTER DEVICE",
+                "PIPE",
+                "SOCKET"};
+        return str_ia_type[type];
+}
+
 gf_boolean_t
 gf_is_zero_filled_stat (struct iatt *buf)
 {
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 620468e8b09..858e574dd4c 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -37,6 +37,7 @@ void trap (void);
 #include "locking.h"
 #include "mem-pool.h"
 #include "compat-uuid.h"
+#include "iatt.h"
 #include "uuid.h"
 #include "libglusterfs-messages.h"
 
@@ -826,4 +827,9 @@ gf_is_zero_filled_stat (struct iatt *buf);
 void
 gf_zero_fill_stat (struct iatt *buf);
 
+gf_boolean_t
+is_virtual_xattr (const char *k);
+
+const char *
+gf_inode_type_to_str (ia_type_t type);
 #endif /* _COMMON_UTILS_H */
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 128dcd32e5f..0cee2ba3868 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -177,6 +177,8 @@
 #define GF_XATTROP_DIRTY_COUNT "glusterfs.xattrop_dirty_count"
 #define GF_XATTROP_ENTRY_IN_KEY "glusterfs.xattrop-entry-create"
 #define GF_XATTROP_ENTRY_OUT_KEY "glusterfs.xattrop-entry-delete"
+#define GF_INDEX_IA_TYPE_GET_REQ "glusterfs.index-ia-type-get-req"
+#define GF_INDEX_IA_TYPE_GET_RSP "glusterfs.index-ia-type-get-rsp"
 
 #define GF_HEAL_INFO "glusterfs.heal-info"
 #define GF_AFR_HEAL_SBRAIN "glusterfs.heal-sbrain"
diff --git a/tests/basic/afr/granular-esh/granular-esh.t b/tests/basic/afr/granular-esh/granular-esh.t
index 4adcfc7456a..ee53878e004 100644
--- a/tests/basic/afr/granular-esh/granular-esh.t
+++ b/tests/basic/afr/granular-esh/granular-esh.t
@@ -149,10 +149,9 @@ TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/dir
 TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/newdir
 TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/file_stale
 
-# To be uncommented once index xl is made to purge the parent gfid dir on rm -rf
-#TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_dir/f1
-#TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_dir/f2
-#TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_dir/f3
+TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_dir/f1
+TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_dir/f2
+TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_dir/f3
 
 TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_newdir/f1
 TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_newdir/f2
@@ -164,7 +163,6 @@ TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_newsubdir/f3
 TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID
 TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_newdir
 TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_newsubdir
-# To be uncommented once index xl is made to purge the parent gfid dir on rm -rf
-#TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_dir
+TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$gfid_dir
 
 cleanup
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index c9a27bf8e20..985cebe76b9 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -567,7 +567,7 @@ afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this,
 
                 if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) {
                         ret = afr_shd_index_purge (subvol, parent_idx_inode,
-                                                   name);
+                                                   name, inode->ia_type);
                         /* Why is ret force-set to 0? We do not care about
                          * index purge failing for full heal as it is quite
                          * possible during replace-brick that not all files
@@ -748,7 +748,11 @@ afr_selfheal_entry_granular_dirent (xlator_t *subvol, gf_dirent_t *entry,
 
         ret = syncop_lookup (args->xl, &loc, &iatt, NULL, NULL, NULL);
         if ((ret == -ENOENT) || (ret == -ESTALE)) {
-                afr_shd_index_purge (subvol, parent->inode, entry->d_name);
+                /* The name indices under the pgfid index dir are guaranteed
+                 * to be regular files. Hence the hardcoding.
+                 */
+                afr_shd_index_purge (subvol, parent->inode, entry->d_name,
+                                     IA_IFREG);
                 ret = 0;
                 goto out;
         }
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 545ffa0a1d7..26f4a80777f 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -153,10 +153,13 @@ unlock:
 inode_t *
 afr_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
 {
-	inode_t *inode = NULL;
-	int ret = 0;
-	loc_t loc = {0, };
-	struct iatt iatt = {0, };
+	int           ret      = 0;
+        uint64_t       val     = IA_INVAL;
+	loc_t         loc      = {0, };
+        dict_t       *xdata    = NULL;
+        dict_t       *rsp_dict = NULL;
+	inode_t      *inode    = NULL;
+	struct iatt   iatt     = {0, };
 
 	inode = inode_find (this->itable, gfid);
 	if (inode)
@@ -167,12 +170,32 @@ afr_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
 		goto out;
 	gf_uuid_copy (loc.gfid, gfid);
 
-	ret = syncop_lookup (subvol, &loc, &iatt, NULL, NULL, NULL);
+        xdata = dict_new ();
+        if (!xdata)
+                goto out;
+
+        ret = dict_set_int8 (xdata, GF_INDEX_IA_TYPE_GET_REQ, 1);
+        if (ret)
+                goto out;
+
+	ret = syncop_lookup (subvol, &loc, &iatt, NULL, xdata, &rsp_dict);
 	if (ret < 0)
 		goto out;
 
+        if (rsp_dict) {
+                ret = dict_get_uint64 (rsp_dict, GF_INDEX_IA_TYPE_GET_RSP,
+                                       &val);
+                if (ret)
+                        goto out;
+        }
+
 	inode = inode_link (loc.inode, NULL, NULL, &iatt);
+        ret = inode_ctx_set2 (inode, subvol, 0, &val);
 out:
+        if (xdata)
+                dict_unref (xdata);
+        if (rsp_dict)
+                dict_unref (rsp_dict);
 	loc_wipe (&loc);
 	return inode;
 }
@@ -215,15 +238,19 @@ out:
 }
 
 int
-afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
+afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name,
+                     ia_type_t type)
 {
-	loc_t loc = {0, };
-	int ret = 0;
+	int    ret = 0;
+	loc_t  loc = {0,};
 
 	loc.parent = inode_ref (inode);
 	loc.name = name;
 
-	ret = syncop_unlink (subvol, &loc, NULL, NULL);
+        if (IA_ISDIR (type))
+                ret = syncop_rmdir (subvol, &loc, 1, NULL, NULL);
+        else
+                ret = syncop_unlink (subvol, &loc, NULL, NULL);
 
 	loc_wipe (&loc);
 	return ret;
@@ -395,6 +422,7 @@ afr_shd_index_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
         afr_private_t        *priv   = NULL;
         uuid_t               gfid    = {0};
         int                  ret     = 0;
+        uint64_t             val     = IA_INVAL;
 
         priv = healer->this->private;
         if (!priv->shd.enabled)
@@ -407,10 +435,13 @@ afr_shd_index_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
         if (ret)
                 return 0;
 
+        inode_ctx_get2 (parent->inode, subvol, NULL, &val);
+
         ret = afr_shd_selfheal (healer, healer->subvol, gfid);
 
         if (ret == -ENOENT || ret == -ESTALE)
-                afr_shd_index_purge (subvol, parent->inode, entry->d_name);
+                afr_shd_index_purge (subvol, parent->inode, entry->d_name, val);
+
         if (ret == 2)
                 /* If bricks crashed in pre-op after creating indices/xattrop
                  * link but before setting afr changelogs, we end up with stale
@@ -489,6 +520,10 @@ afr_shd_index_sweep_all (struct subvol_healer *healer)
                 goto out;
         count += ret;
 
+        ret = afr_shd_index_sweep (healer, GF_XATTROP_ENTRY_CHANGES_GFID);
+        if (ret < 0)
+                goto out;
+        count += ret;
 out:
         if (ret < 0)
                 return ret;
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index f591515669c..c6ac5ebfd1b 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -75,5 +75,6 @@ afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid,
                       char **path_p);
 
 int
-afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name);
+afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name,
+                     ia_type_t type);
 #endif /* !_AFR_SELF_HEALD_H */
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index cd40f2587b2..4f431648913 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -13,6 +13,7 @@
 #include "syscall.h"
 #include "syncop.h"
 #include "common-utils.h"
+#include <ftw.h>
 
 #define XATTROP_SUBDIR "xattrop"
 #define DIRTY_SUBDIR "dirty"
@@ -21,6 +22,7 @@
 struct index_syncop_args {
         inode_t *parent;
         gf_dirent_t *entries;
+        char *path;
 };
 
 static char *index_vgfid_xattrs[XATTROP_TYPE_END] = {
@@ -1482,6 +1484,7 @@ index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
         int             ret = 0;
         int32_t         op_errno = EINVAL;
         int32_t         op_ret = -1;
+        uint64_t        val = IA_INVAL;
         char            path[PATH_MAX] = {0};
         struct iatt     stbuf        = {0, };
         struct iatt     postparent = {0,};
@@ -1509,6 +1512,15 @@ index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
                 make_index_dir_path (priv->index_basepath, subdir,
                                      path, sizeof (path));
                 is_dir = _gf_true;
+
+                if ((xattr_req) &&
+                    (dict_get (xattr_req, GF_INDEX_IA_TYPE_GET_REQ))) {
+                        if (0 == strcmp (subdir,
+                                    index_get_subdir_from_type(ENTRY_CHANGES)))
+                                val = IA_IFDIR;
+                        else
+                                val = IA_IFREG;
+                }
         } else {
                 if (!inode_is_linked (loc->inode)) {
                         inode_unref (iloc.inode);
@@ -1524,12 +1536,12 @@ index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
         ret = sys_lstat (path, &lstatbuf);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG, "Stat failed on %s dir "
-                        "(%s)", subdir, strerror (errno));
+                        "(%s)", path, strerror (errno));
                 op_errno = errno;
                 goto done;
         } else if (!S_ISDIR (lstatbuf.st_mode) && is_dir) {
                 gf_log (this->name, GF_LOG_DEBUG, "Stat failed on %s dir, "
-                        "not a directory", subdir);
+                        "not a directory", path);
                 op_errno = ENOENT;
                 goto done;
         }
@@ -1539,6 +1551,15 @@ index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
                 goto done;
         }
 
+        if (val != IA_INVAL) {
+                ret = dict_set_uint64 (xattr, GF_INDEX_IA_TYPE_GET_RSP, val);
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = -ret;
+                        goto done;
+                }
+        }
+
         iatt_from_stat (&stbuf, &lstatbuf);
         if (is_dir || inode_is_linked (iloc.inode))
                 loc_gfid (&iloc, stbuf.ia_gfid);
@@ -1654,6 +1675,116 @@ done:
 }
 
 int
+deletion_handler (const char *fpath, const struct stat *sb, int typeflag,
+               struct FTW *ftwbuf)
+{
+        int          ret  = -1;
+        ia_type_t    type = IA_INVAL;
+
+        switch (sb->st_mode & S_IFMT) {
+        case S_IFREG:
+                sys_unlink (fpath);
+                break;
+
+        case S_IFDIR:
+                sys_rmdir (fpath);
+                break;
+        default:
+                type = ia_type_from_st_mode (sb->st_mode);
+                gf_log (THIS->name, GF_LOG_WARNING, "%s neither a regular file "
+                        "nor a directory - type:%s", fpath,
+                        gf_inode_type_to_str (type));
+                break;
+        }
+        return 0;
+}
+
+static int
+index_wipe_index_subdir (void *opaque)
+{
+        struct index_syncop_args *args  = opaque;
+
+        nftw (args->path, deletion_handler, 1, FTW_DEPTH | FTW_PHYS);
+        return 0;
+}
+
+static void
+index_get_parent_iatt (struct iatt *parent, char *path, loc_t *loc,
+                       int32_t *op_ret, int32_t *op_errno)
+{
+        int         ret      = -1;
+        struct stat lstatbuf = {0,};
+
+        ret = sys_lstat (path, &lstatbuf);
+        if (ret < 0) {
+                *op_ret = -1;
+                *op_errno = errno;
+                return;
+        }
+
+        iatt_from_stat (parent, &lstatbuf);
+        gf_uuid_copy (parent->ia_gfid, loc->pargfid);
+        parent->ia_ino = -1;
+
+        return;
+}
+
+int
+index_rmdir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                     dict_t *xdata)
+{
+        int                        ret                    = 0;
+        int32_t                    op_ret                 = 0;
+        int32_t                    op_errno               = 0;
+        char                      *subdir                 = NULL;
+        char                       index_dir[PATH_MAX]    = {0};
+        char                       index_subdir[PATH_MAX] = {0};
+        uuid_t                     gfid                   = {0};
+        struct  iatt               preparent              = {0};
+        struct  iatt               postparent             = {0};
+        index_priv_t              *priv                   = NULL;
+        index_xattrop_type_t       type                   = XATTROP_TYPE_UNSET;
+        struct index_syncop_args   args                   = {0,};
+
+        priv = this->private;
+
+        type = index_get_type_from_vgfid (priv, loc->pargfid);
+        subdir = index_get_subdir_from_vgfid (priv, loc->pargfid);
+        make_index_dir_path (priv->index_basepath, subdir,
+                             index_dir, sizeof (index_dir));
+
+        index_get_parent_iatt (&preparent, index_dir, loc, &op_ret, &op_errno);
+        if (op_ret < 0)
+                goto done;
+
+        gf_uuid_parse (loc->name, gfid);
+        make_gfid_path (priv->index_basepath, subdir, gfid, index_subdir,
+                        sizeof (index_subdir));
+
+        if (flag == 0) {
+                ret = index_del (this, gfid, subdir, type);
+                if (ret < 0) {
+                        op_ret = -1;
+                        op_errno = -ret;
+                        goto done;
+                }
+        } else {
+                args.path = index_subdir;
+                ret = synctask_new (this->ctx->env, index_wipe_index_subdir,
+                                    NULL, NULL, &args);
+        }
+
+        index_get_parent_iatt (&postparent, index_dir, loc, &op_ret, &op_errno);
+        if (op_ret < 0)
+                goto done;
+
+done:
+        INDEX_STACK_UNWIND (rmdir, frame, op_ret, op_errno, &preparent,
+                            &postparent, xdata);
+        return 0;
+}
+
+int
 index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
                       dict_t *xdata)
 {
@@ -1667,7 +1798,6 @@ index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
         struct  iatt    postparent = {0};
         char            index_dir[PATH_MAX] = {0};
         char            filepath[PATH_MAX] = {0};
-        struct  stat    lstatbuf = {0};
         uuid_t          gfid = {0};
         char            *subdir = NULL;
 
@@ -1681,16 +1811,9 @@ index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
                 goto done;
         }
 
-        ret = sys_lstat (index_dir, &lstatbuf);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = errno;
+        index_get_parent_iatt (&preparent, index_dir, loc, &op_ret, &op_errno);
+        if (op_ret < 0)
                 goto done;
-        }
-
-        iatt_from_stat (&preparent, &lstatbuf);
-        gf_uuid_copy (preparent.ia_gfid, loc->pargfid);
-        preparent.ia_ino = -1;
 
         if (type <= XATTROP_TYPE_UNSET) {
                 ret = index_inode_ctx_get (loc->parent, this, &ictx);
@@ -1716,16 +1839,9 @@ index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
                 goto done;
         }
 
-        memset (&lstatbuf, 0, sizeof (lstatbuf));
-        ret = sys_lstat (index_dir, &lstatbuf);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = errno;
+        index_get_parent_iatt (&postparent, index_dir, loc, &op_ret, &op_errno);
+        if (op_ret < 0)
                 goto done;
-        }
-        iatt_from_stat (&postparent, &lstatbuf);
-        gf_uuid_copy (postparent.ia_gfid, loc->pargfid);
-        postparent.ia_ino = -1;
 done:
         INDEX_STACK_UNWIND (unlink, frame, op_ret, op_errno, &preparent,
                             &postparent, xdata);
@@ -2010,6 +2126,31 @@ out:
 }
 
 int
+index_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (!index_is_fop_on_internal_inode (this, loc->parent, NULL))
+                goto out;
+
+        stub = fop_rmdir_stub (frame, index_rmdir_wrapper, loc, flags, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL,
+                                     NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+        return 0;
+}
+
+int
 index_make_xattrop_watchlist (xlator_t *this, index_priv_t *priv,
                               char *watchlist, index_xattrop_type_t type)
 {
@@ -2333,6 +2474,7 @@ struct xlator_fops fops = {
         .opendir     = index_opendir,
         .readdir     = index_readdir,
         .unlink      = index_unlink,
+        .rmdir       = index_rmdir,
         .fstat       = index_fstat,
 };