features/quota: Metadata cleanup

Quota and marker uses 'trusted.glusterfs.quota*' and 'trusted.pgfid*' xattrs to
store its configurations and accounting information and also to build the
parent inode chain in case of absense of path.

Problem:
After disabling and then enabling quota back, the xattrs may contain stale data
leading to impaired accounting and thus improper enforcement.

Solution:
Clean up all the quota related xattrs after quota disable.

Marker xlator implements a virtual xattr to cleanup quota and pgfid xattrs. In
this approach glusterd mounts an auxiliary mount and sends the below command to
all the files by crawling the mountpoint.

 #setfattr -n "glusterfs.quota-xattr-cleanup" -v 1 <path/to/file>

Credit:
Krishnan Parthasarathi <kparthas@redhat.com>
Varun Shastry          <vshastry@redhat.com>
Change-Id: I9380eca58a285dc27dd572de1767aac8f2cd8049
BUG: 969461
Signed-off-by: Varun Shastry <vshastry@redhat.com>
Reviewed-on: http://review.gluster.org/6369
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 164 insertions, 9 deletions

diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index 280d43ae38b..d6b6dd358de 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -21,6 +21,7 @@
 #include "marker-quota-helper.h"
 #include "marker-common.h"
 #include "byte-order.h"
+#include "syncop.h"
 
 #define _GF_UID_GID_CHANGED 1
 
@@ -2153,16 +2154,150 @@ out:
         return 0;
 }
 
+int
+remove_quota_keys (dict_t *dict, char *k, data_t *v, void *data)
+{
+	call_frame_t 	*frame = data;
+	marker_local_t 	*local = frame->local;
+	xlator_t 	*this = frame->this;
+	int 		ret = -1;
+
+	ret = syncop_removexattr (FIRST_CHILD (this), &local->loc, k);
+	if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove "
+			"extended attribute: %s", local->loc.path, k);
+                return -1;
+	}
+	return 0;
+}
+
+int
+quota_xattr_cleaner_cbk (int ret, call_frame_t *frame, void *args)
+{
+        dict_t *xdata = args;
+        int op_ret = -1;
+        int op_errno = 0;
+	marker_local_t *local = NULL;
+
+	local = frame->local;
+	frame->local = NULL;
+
+        op_ret   = (ret < 0)? -1: 0;
+        op_errno = -ret;
+
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+	marker_local_unref (local);
+        return ret;
+}
+
+int
+quota_xattr_cleaner (void *args)
+{
+        struct synctask *task  = NULL;
+        call_frame_t    *frame = NULL;
+        xlator_t        *this  = NULL;
+        marker_local_t   *local = NULL;
+        dict_t          *xdata = NULL;
+        int             ret    = -1;
+
+        task = synctask_get ();
+        if (!task)
+                goto out;
+
+        frame = task->frame;
+        this  = frame->this;
+        local = frame->local;
+
+        ret = syncop_listxattr (FIRST_CHILD(this), &local->loc, &xdata);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+
+	ret = dict_foreach_fnmatch (xdata, "trusted.glusterfs.quota.*",
+                                    remove_quota_keys, frame);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+	ret = dict_foreach_fnmatch (xdata, PGFID_XATTR_KEY_PREFIX"*",
+                                    remove_quota_keys, frame);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (xdata)
+                dict_unref (xdata);
+
+        return ret;
+}
+
+int
+marker_do_xattr_cleanup (call_frame_t *frame, xlator_t *this, dict_t *xdata,
+                        loc_t *loc)
+{
+        int           ret       = -1;
+        marker_local_t *local    = NULL;
+
+        local = mem_get0 (this->local_pool);
+	if (!local)
+		goto out;
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        loc_copy (&local->loc, loc);
+        ret = synctask_new (this->ctx->env, quota_xattr_cleaner,
+			    quota_xattr_cleaner_cbk, frame, xdata);
+        if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "Failed to create synctask "
+			"for cleaning up quota extended attributes");
+                goto out;
+	}
+
+        ret = 0;
+out:
+        if (ret) {
+		frame->local = NULL;
+                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, xdata);
+		marker_local_unref (local);
+        }
+        return ret;
+}
+
+static inline gf_boolean_t
+marker_xattr_cleanup_cmd (dict_t *dict)
+{
+        return (dict_get (dict, VIRTUAL_QUOTA_XATTR_CLEANUP_KEY) != NULL);
+}
+
 int32_t
 marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
                  int32_t flags, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+        int32_t          ret            = 0;
+        marker_local_t  *local          = NULL;
+        marker_conf_t   *priv           = NULL;
+        int              op_errno       = ENOMEM;
 
         priv = this->private;
 
+        if (marker_xattr_cleanup_cmd (dict)) {
+                if (frame->root->uid != 0 || frame->root->gid != 0) {
+                        op_errno = EPERM;
+                        ret = -1;
+                        goto err;
+                }
+
+                /* The following function does the cleanup and then unwinds the
+                 * corresponding call*/
+                loc_path (loc, NULL);
+                marker_do_xattr_cleanup (frame, this, xdata, loc);
+                return 0;
+        }
+
         if (priv->feature_enabled == 0)
                 goto wind;
 
@@ -2183,7 +2318,7 @@ wind:
                     FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
+        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index 3c8dcf8dd1c..ffde532bdcd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -28,6 +28,8 @@
 #include <sys/wait.h>
 #include <dlfcn.h>
 
+/* Any negative pid to make it special client */
+#define QUOTA_CRAWL_PID "-100"
 
 const char *gd_quota_op_list[GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT+1] = {
         [GF_QUOTA_OPTION_TYPE_NONE]               = "none",
@@ -158,7 +160,8 @@ out:
 }
 
 int32_t
-glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
+glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
+                                  int type)
 {
         pid_t                      pid;
         int32_t                    ret              = 0;
@@ -178,6 +181,7 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
                          "-s", "localhost",
                          "--volfile-id", volname,
 			 "--use-readdirp=no",
+                         "--client-pid", QUOTA_CRAWL_PID,
                          "-l", DEFAULT_LOG_FILE_DIRECTORY"/quota-crawl.log",
                          mountdir, NULL);
 
@@ -210,7 +214,19 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
                         exit (EXIT_FAILURE);
                 }
                 runinit (&runner);
-                runner_add_args (&runner, "/usr/bin/find", "find", ".", NULL);
+
+                if (type == GF_QUOTA_OPTION_TYPE_ENABLE)
+
+                        runner_add_args (&runner, "/usr/bin/find", "find", ".",
+                                         NULL);
+
+                else if (type == GF_QUOTA_OPTION_TYPE_DISABLE)
+
+                        runner_add_args (&runner, "/usr/bin/find", ".",
+                                         "-exec", "/usr/bin/setfattr", "-n",
+                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "-v",
+                                         "1", "{}", "\\", ";", NULL);
+
                 if (runner_start (&runner) == -1)
                         _exit (EXIT_FAILURE);
 
@@ -325,7 +341,8 @@ out:
 }
 
 int32_t
-glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
+glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr,
+                        gf_boolean_t *crawl)
 {
         int32_t    ret            = -1;
         int        i              =  0;
@@ -381,6 +398,8 @@ glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
         if (ret)
                 goto out;
 
+        *crawl = _gf_true;
+
         (void) glusterd_clean_up_quota_store (volinfo);
 
         ret = 0;
@@ -1045,7 +1064,8 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                         break;
 
                 case GF_QUOTA_OPTION_TYPE_DISABLE:
-                        ret = glusterd_quota_disable (volinfo, op_errstr);
+                        ret = glusterd_quota_disable (volinfo, op_errstr,
+                                                      &start_crawl);
                         if (ret < 0)
                                 goto out;
 
@@ -1129,7 +1149,7 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         }
 
         if (rsp_dict && start_crawl == _gf_true)
-                glusterd_quota_initiate_fs_crawl (priv, volname);
+                glusterd_quota_initiate_fs_crawl (priv, volname, type);
 
         if (priv->op_version > GD_OP_VERSION_MIN) {
                 ret = glusterd_quotad_op (type);