cluster/dht : User xattrs are not healed after brick stop/start

Problem: In a distributed volume custom extended attribute value for a directory
         does not display correct value after stop/start or added newly brick.
         If any extended(acl) attribute value is set for a directory after stop/added
         the brick the attribute(user|acl|quota) value is not updated on brick
         after start the brick.

Solution: First store hashed subvol or subvol(has internal xattr) on inode ctx and
          consider it as a MDS subvol.At the time of update custom xattr
          (user,quota,acl, selinux) on directory first check the mds from
          inode ctx, if mds is not present on inode ctx then throw EINVAL error
          to application otherwise set xattr on MDS subvol with internal xattr
          value of -1 and then try to update the attribute on other non MDS
          volumes also.If mds subvol is down in that case throw an
          error "Transport endpoint is not connected". In dht_dir_lookup_cbk|
          dht_revalidate_cbk|dht_discover_complete call dht_call_dir_xattr_heal
          to heal custom extended attribute.
          In case of gnfs server if hashed subvol has not found based on
          loc then wind a call on all subvol to update xattr.

Fix:    1) Save MDS subvol on inode ctx
        2) Check if mds subvol is present on inode ctx
        3) If mds subvol is down then call unwind with error ENOTCONN and if it is up
           then set new xattr "GF_DHT_XATTR_MDS" to -1 and wind a call on other
           subvol.
        4) If setxattr fop is successful on non-mds subvol then increment the value of
           internal xattr to +1
        5) At the time of directory_lookup check the value of new xattr GF_DHT_XATTR_MDS
        6) If value is not 0 in dht_lookup_dir_cbk(other cbk) functions then call heal
           function to heal user xattr
        7) syncop_setxattr on hashed_subvol to reset the value of xattr to 0
           if heal is successful on all subvol.

Test : To reproduce the issue followed below steps
       1) Create a distributed volume and create mount point
       2) Create some directory from mount point mkdir tmp{1..5}
       3) Kill any one brick from the volume
       4) Set extended attribute from mount point on directory
          setfattr -n user.foo -v "abc" ./tmp{1..5}
          It will throw error " Transport End point is not connected "
          for those hashed subvol is down
       5) Start volume with force option to start brick process
       6) Execute getfattr command on mount point for directory
       7) Check extended attribute on brick
          getfattr -n user.foo <volume-location>/tmp{1..5}
          It shows correct value for directories for those
          xattr fop were executed successfully.

Note: The patch will resolve xattr healing problem only for fuse mount
      not for nfs mount.

BUG: 1371806
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Change-Id: I4eb137eace24a8cb796712b742f1d177a65343d5

1 files changed, 62 insertions, 11 deletions

diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index d06b7314d8b..9b39596f872 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -19,6 +19,7 @@
 #include "refcount.h"
 #include "timer.h"
 #include "protocol-common.h"
+#include "glusterfs-acl.h"
 
 #ifndef _DHT_H
 #define _DHT_H
@@ -26,6 +27,7 @@
 #define GF_XATTR_FIX_LAYOUT_KEY         "distribute.fix.layout"
 #define GF_XATTR_TIER_LAYOUT_FIXED_KEY  "trusted.tier.fix.layout.complete"
 #define GF_XATTR_FILE_MIGRATE_KEY       "trusted.distribute.migrate-data"
+#define DHT_MDS_STR                     "mds"
 #define GF_DHT_LOOKUP_UNHASHED_ON       1
 #define GF_DHT_LOOKUP_UNHASHED_AUTO     2
 #define DHT_PATHINFO_HEADER             "DISTRIBUTE:"
@@ -41,6 +43,12 @@
 #define DHT_DIR_STAT_BLOCKS          8
 #define DHT_DIR_STAT_SIZE            4096
 
+/* Array to hold custom xattr keys
+*/
+extern char *xattrs_to_heal[];
+
+
+
 #include <fnmatch.h>
 
 typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
@@ -107,6 +115,7 @@ struct dht_inode_ctx {
         dht_layout_t    *layout;
         dht_stat_time_t  time;
         xlator_t        *lock_subvol;
+        xlator_t        *mds_subvol;     /* This is only used for directories */
 };
 
 typedef struct dht_inode_ctx dht_inode_ctx_t;
@@ -252,6 +261,7 @@ struct dht_local {
         /* Use stbuf as the postbuf, when we require both
          * pre and post attrs */
         struct iatt              stbuf;
+        struct iatt              mds_stbuf;
         struct iatt              prebuf;
         struct iatt              preoldparent;
         struct iatt              postoldparent;
@@ -263,6 +273,8 @@ struct dht_local {
         inode_t                 *inode;
         dict_t                  *params;
         dict_t                  *xattr;
+        dict_t                  *mds_xattr;
+        dict_t                  *xdata;      /* dict used to save xdata response by xattr fop */
         dict_t                  *xattr_req;
         dht_layout_t            *layout;
         size_t                   size;
@@ -271,7 +283,9 @@ struct dht_local {
         xlator_t                *dst_hashed, *dst_cached;
         xlator_t                *cached_subvol;
         xlator_t                *hashed_subvol;
+        xlator_t                *mds_subvol; /* This is use for dir only */
         char                     need_selfheal;
+        char                     need_xattr_heal;
         int                      file_count;
         int                      dir_count;
         call_frame_t            *main_frame;
@@ -355,6 +369,9 @@ struct dht_local {
 
         /* fd open check */
         gf_boolean_t fd_checked;
+        /* This is use only for directory operation */
+        int32_t valid;
+        gf_boolean_t heal_layout;
 };
 typedef struct dht_local dht_local_t;
 
@@ -629,6 +646,7 @@ struct dht_conf {
 
         /* Support variable xattr names. */
         char            *xattr_name;
+        char            *mds_xattr_key;
         char            *link_xattr_name;
         char            *commithash_xattr_name;
         char            *wild_xattr_name;
@@ -1311,9 +1329,6 @@ dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
 int
 add_opt(char **optsp, const char *opt);
 
-char *
-getChoices (const char *value);
-
 int
 dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value);
 
@@ -1323,18 +1338,12 @@ dht_remove_stale_linkto (void *data);
 int
 dht_remove_stale_linkto_cbk (int ret, call_frame_t *sync_frame, void *data);
 
-
 int
 dht_fd_ctx_set (xlator_t *this, fd_t *fd, xlator_t *subvol);
 
 int
 dht_check_and_open_fd_on_subvol (xlator_t *this, call_frame_t *frame);
 
-
-
-
-
-
 /* FD fop callbacks */
 
 int
@@ -1387,13 +1396,55 @@ int
 dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata);
 
-
 int
 dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int op_ret, int op_errno, dict_t *xdata);
 
-
 int
 dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno, dict_t *xdata);
+
+/* All custom xattr heal functions */
+int
+dht_dir_heal_xattrs (void *data);
+
+int
+dht_dir_heal_xattrs_done (int ret, call_frame_t *sync_frame, void *data);
+
+void
+dht_aggregate_xattr (dict_t *dst, dict_t *src);
+
+int32_t
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size);
+
+int
+dht_set_user_xattr (dict_t *dict, char *k, data_t *v, void *data);
+
+void
+dht_dir_set_heal_xattr (xlator_t *this, dht_local_t *local, dict_t *dst,
+                        dict_t *src, int *uret, int *uflag);
+
+int
+dht_dir_xattr_heal (xlator_t *this, dht_local_t *local);
+
+int32_t
+dht_dict_get_array (dict_t *dict, char *key, int32_t value[], int32_t size, int *errst);
+
+xlator_t *
+dht_inode_get_hashed_subvol (inode_t *inode, xlator_t *this, loc_t *loc);
+
+int
+dht_mark_mds_subvolume (call_frame_t *frame, xlator_t *this);
+
+int
+dht_mds_internal_setxattr_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int op_ret, int op_errno,
+                               dict_t *xdata);
+int
+dht_inode_ctx_mdsvol_set (inode_t *inode, xlator_t *this,
+                          xlator_t *mds_subvol);
+int
+dht_inode_ctx_mdsvol_get (inode_t *inode, xlator_t *this,
+                          xlator_t **mdsvol);
+
 #endif/* _DHT_H */