features/quota: Metadata cleanup

Quota and marker uses 'trusted.glusterfs.quota*' and 'trusted.pgfid*' xattrs to
store its configurations and accounting information and also to build the
parent inode chain in case of absense of path.

Problem:
After disabling and then enabling quota back, the xattrs may contain stale data
leading to impaired accounting and thus improper enforcement.

Solution:
Clean up all the quota related xattrs after quota disable.

Marker xlator implements a virtual xattr to cleanup quota and pgfid xattrs. In
this approach glusterd mounts an auxiliary mount and sends the below command to
all the files by crawling the mountpoint.

 #setfattr -n "glusterfs.quota-xattr-cleanup" -v 1 <path/to/file>

Credit:
Krishnan Parthasarathi <kparthas@redhat.com>
Varun Shastry          <vshastry@redhat.com>
Change-Id: I9380eca58a285dc27dd572de1767aac8f2cd8049
BUG: 969461
Signed-off-by: Varun Shastry <vshastry@redhat.com>
Reviewed-on: http://review.gluster.org/6369
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

6 files changed, 169 insertions, 63 deletions

diff --git a/extras/Makefile.am b/extras/Makefile.am
index cf619329b75..2633e20a571 100644
--- a/extras/Makefile.am
+++ b/extras/Makefile.am
@@ -1,4 +1,3 @@
-
 EditorModedir = $(docdir)
 EditorMode_DATA = glusterfs-mode.el glusterfs.vim
 
@@ -13,8 +12,7 @@ vol_DATA = glusterd.vol
 
 EXTRA_DIST = specgen.scm MacOSX/Portfile glusterfs-mode.el glusterfs.vim  \
 	migrate-unify-to-distribute.sh backend-xattr-sanitize.sh          \
-	backend-cleanup.sh disk_usage_sync.sh quota-remove-xattr.sh       \
-	quota-metadata-cleanup.sh glusterfs-logrotate clear_xattrs.sh     \
-	group-virt.example glusterd-sysconfig gluster-rsyslog-7.2.conf    \
-	gluster-rsyslog-5.8.conf logger.conf.example glusterd.vol         \
-	glusterfs-georep-logrotate
+	backend-cleanup.sh disk_usage_sync.sh glusterfs-logrotate         \
+        clear_xattrs.sh group-virt.example glusterd-sysconfig             \
+        gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf                 \
+        logger.conf.example glusterd.vol glusterfs-georep-logrotate
diff --git a/extras/quota-metadata-cleanup.sh b/extras/quota-metadata-cleanup.sh
deleted file mode 100755
index 37ad8bc3137..00000000000
--- a/extras/quota-metadata-cleanup.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# This script is used to cleanup xattrs setup by quota-translator in (a)
-# backend directory(ies). It takes a single or list of backend directories
-# as argument(s).
-
-usage ()
-{
-    echo >&2 "usage: $0 <list-of-backend-directories>"
-}
-
-main ()
-{
-    [ $# -lt 1 ] && usage
-
-    INSTALL_DIR=`dirname $0`
-
-    for i in $@; do
-        find $i -exec $INSTALL_DIR/quota-remove-xattr.sh '{}' \;
-    done
-    
-}
-
-main $@
diff --git a/extras/quota-remove-xattr.sh b/extras/quota-remove-xattr.sh
deleted file mode 100755
index 7191f9bd443..00000000000
--- a/extras/quota-remove-xattr.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# This script is used to remove xattrs set by quota translator on a path.
-# It is generally invoked from quota-metadata-cleanup.sh, but can
-# also be used stand-alone.
-
-usage ()
-{
-    echo >&2 "usage: $0 <path>"
-}
-
-main ()
-{
-    [ $# -ne 1 ] && usage $0
-
-    XATTR_KEY_VALUE_PAIRS=`getfattr -h -d -m 'trusted.glusterfs.quota' $1 2>/dev/null | sed -e '/^# file/d'`
-
-    for i in $XATTR_KEY_VALUE_PAIRS; do
-        XATTR_KEY=`echo $i | sed -e 's/\([^=]*\).*/\1/g'`
-        setfattr -h -x $XATTR_KEY $1
-    done
-}
-
-main $@
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 6dc2fe6df8a..09b26ecf331 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -87,6 +87,7 @@
 #define GF_XATTR_GET_REAL_FILENAME_KEY "glusterfs.get_real_filename:"
 #define GF_XATTR_USER_PATHINFO_KEY   "glusterfs.pathinfo"
 #define QUOTA_LIMIT_KEY "trusted.glusterfs.quota.limit-set"
+#define VIRTUAL_QUOTA_XATTR_CLEANUP_KEY "glusterfs.quota-xattr-cleanup"
 
 #define GF_READDIR_SKIP_DIRS       "readdir-filter-directories"
 
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index 280d43ae38b..d6b6dd358de 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -21,6 +21,7 @@
 #include "marker-quota-helper.h"
 #include "marker-common.h"
 #include "byte-order.h"
+#include "syncop.h"
 
 #define _GF_UID_GID_CHANGED 1
 
@@ -2153,16 +2154,150 @@ out:
         return 0;
 }
 
+int
+remove_quota_keys (dict_t *dict, char *k, data_t *v, void *data)
+{
+	call_frame_t 	*frame = data;
+	marker_local_t 	*local = frame->local;
+	xlator_t 	*this = frame->this;
+	int 		ret = -1;
+
+	ret = syncop_removexattr (FIRST_CHILD (this), &local->loc, k);
+	if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove "
+			"extended attribute: %s", local->loc.path, k);
+                return -1;
+	}
+	return 0;
+}
+
+int
+quota_xattr_cleaner_cbk (int ret, call_frame_t *frame, void *args)
+{
+        dict_t *xdata = args;
+        int op_ret = -1;
+        int op_errno = 0;
+	marker_local_t *local = NULL;
+
+	local = frame->local;
+	frame->local = NULL;
+
+        op_ret   = (ret < 0)? -1: 0;
+        op_errno = -ret;
+
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+	marker_local_unref (local);
+        return ret;
+}
+
+int
+quota_xattr_cleaner (void *args)
+{
+        struct synctask *task  = NULL;
+        call_frame_t    *frame = NULL;
+        xlator_t        *this  = NULL;
+        marker_local_t   *local = NULL;
+        dict_t          *xdata = NULL;
+        int             ret    = -1;
+
+        task = synctask_get ();
+        if (!task)
+                goto out;
+
+        frame = task->frame;
+        this  = frame->this;
+        local = frame->local;
+
+        ret = syncop_listxattr (FIRST_CHILD(this), &local->loc, &xdata);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+
+	ret = dict_foreach_fnmatch (xdata, "trusted.glusterfs.quota.*",
+                                    remove_quota_keys, frame);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+	ret = dict_foreach_fnmatch (xdata, PGFID_XATTR_KEY_PREFIX"*",
+                                    remove_quota_keys, frame);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (xdata)
+                dict_unref (xdata);
+
+        return ret;
+}
+
+int
+marker_do_xattr_cleanup (call_frame_t *frame, xlator_t *this, dict_t *xdata,
+                        loc_t *loc)
+{
+        int           ret       = -1;
+        marker_local_t *local    = NULL;
+
+        local = mem_get0 (this->local_pool);
+	if (!local)
+		goto out;
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        loc_copy (&local->loc, loc);
+        ret = synctask_new (this->ctx->env, quota_xattr_cleaner,
+			    quota_xattr_cleaner_cbk, frame, xdata);
+        if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "Failed to create synctask "
+			"for cleaning up quota extended attributes");
+                goto out;
+	}
+
+        ret = 0;
+out:
+        if (ret) {
+		frame->local = NULL;
+                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, xdata);
+		marker_local_unref (local);
+        }
+        return ret;
+}
+
+static inline gf_boolean_t
+marker_xattr_cleanup_cmd (dict_t *dict)
+{
+        return (dict_get (dict, VIRTUAL_QUOTA_XATTR_CLEANUP_KEY) != NULL);
+}
+
 int32_t
 marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
                  int32_t flags, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+        int32_t          ret            = 0;
+        marker_local_t  *local          = NULL;
+        marker_conf_t   *priv           = NULL;
+        int              op_errno       = ENOMEM;
 
         priv = this->private;
 
+        if (marker_xattr_cleanup_cmd (dict)) {
+                if (frame->root->uid != 0 || frame->root->gid != 0) {
+                        op_errno = EPERM;
+                        ret = -1;
+                        goto err;
+                }
+
+                /* The following function does the cleanup and then unwinds the
+                 * corresponding call*/
+                loc_path (loc, NULL);
+                marker_do_xattr_cleanup (frame, this, xdata, loc);
+                return 0;
+        }
+
         if (priv->feature_enabled == 0)
                 goto wind;
 
@@ -2183,7 +2318,7 @@ wind:
                     FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
+        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index 3c8dcf8dd1c..ffde532bdcd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -28,6 +28,8 @@
 #include <sys/wait.h>
 #include <dlfcn.h>
 
+/* Any negative pid to make it special client */
+#define QUOTA_CRAWL_PID "-100"
 
 const char *gd_quota_op_list[GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT+1] = {
         [GF_QUOTA_OPTION_TYPE_NONE]               = "none",
@@ -158,7 +160,8 @@ out:
 }
 
 int32_t
-glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
+glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
+                                  int type)
 {
         pid_t                      pid;
         int32_t                    ret              = 0;
@@ -178,6 +181,7 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
                          "-s", "localhost",
                          "--volfile-id", volname,
 			 "--use-readdirp=no",
+                         "--client-pid", QUOTA_CRAWL_PID,
                          "-l", DEFAULT_LOG_FILE_DIRECTORY"/quota-crawl.log",
                          mountdir, NULL);
 
@@ -210,7 +214,19 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
                         exit (EXIT_FAILURE);
                 }
                 runinit (&runner);
-                runner_add_args (&runner, "/usr/bin/find", "find", ".", NULL);
+
+                if (type == GF_QUOTA_OPTION_TYPE_ENABLE)
+
+                        runner_add_args (&runner, "/usr/bin/find", "find", ".",
+                                         NULL);
+
+                else if (type == GF_QUOTA_OPTION_TYPE_DISABLE)
+
+                        runner_add_args (&runner, "/usr/bin/find", ".",
+                                         "-exec", "/usr/bin/setfattr", "-n",
+                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "-v",
+                                         "1", "{}", "\\", ";", NULL);
+
                 if (runner_start (&runner) == -1)
                         _exit (EXIT_FAILURE);
 
@@ -325,7 +341,8 @@ out:
 }
 
 int32_t
-glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
+glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr,
+                        gf_boolean_t *crawl)
 {
         int32_t    ret            = -1;
         int        i              =  0;
@@ -381,6 +398,8 @@ glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
         if (ret)
                 goto out;
 
+        *crawl = _gf_true;
+
         (void) glusterd_clean_up_quota_store (volinfo);
 
         ret = 0;
@@ -1045,7 +1064,8 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                         break;
 
                 case GF_QUOTA_OPTION_TYPE_DISABLE:
-                        ret = glusterd_quota_disable (volinfo, op_errstr);
+                        ret = glusterd_quota_disable (volinfo, op_errstr,
+                                                      &start_crawl);
                         if (ret < 0)
                                 goto out;
 
@@ -1129,7 +1149,7 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         }
 
         if (rsp_dict && start_crawl == _gf_true)
-                glusterd_quota_initiate_fs_crawl (priv, volname);
+                glusterd_quota_initiate_fs_crawl (priv, volname, type);
 
         if (priv->op_version > GD_OP_VERSION_MIN) {
                 ret = glusterd_quotad_op (type);