ctime: Set mdata xattr on legacy files

Problem:
The files which were created before ctime enabled would not
have "trusted.glusterfs.mdata"(stores time attributes) xattr.
Upon fops which modifies either ctime or mtime, the xattr
gets created with latest ctime, mtime and atime, which is
incorrect. It should update only the corresponding time
attribute and rest from backend

Solution:
Creating xattr with values from brick is not possible as
each brick of replica set would have different times.
So create the xattr upon successful lookup if the xattr
is not created

Note To Reviewers:
The time attributes used to set xattr is got from successful
lookup. Instead of sending the whole iatt over the wire via
setxattr, a structure called mdata_iatt is sent. The mdata_iatt
contains only time attributes.

Backport of:
 > Patch:  https://review.gluster.org/22936
 > Change-Id: I5e535631ddef04195361ae0364336410a2895dd4
 > BUG: 1593542
 > Signed-off-by: Kotresh HR <khiremat@redhat.com>

Change-Id: I5e535631ddef04195361ae0364336410a2895dd4
updates: bz#1739430
Signed-off-by: Kotresh HR <khiremat@redhat.com>

6 files changed, 228 insertions, 56 deletions

diff --git a/xlators/features/utime/src/utime-messages.h b/xlators/features/utime/src/utime-messages.h
index bac18aba460..bd40265abaf 100644
--- a/xlators/features/utime/src/utime-messages.h
+++ b/xlators/features/utime/src/utime-messages.h
@@ -23,6 +23,7 @@
  * glfs-message-id.h.
  */
 
-GLFS_MSGID(UTIME, UTIME_MSG_NO_MEMORY);
+GLFS_MSGID(UTIME, UTIME_MSG_NO_MEMORY, UTIME_MSG_SET_MDATA_FAILED,
+           UTIME_MSG_DICT_SET_FAILED);
 
 #endif /* __UTIME_MESSAGES_H__ */
diff --git a/xlators/features/utime/src/utime.c b/xlators/features/utime/src/utime.c
index 877c751c764..2a986e71add 100644
--- a/xlators/features/utime/src/utime.c
+++ b/xlators/features/utime/src/utime.c
@@ -9,8 +9,10 @@
 */
 
 #include "utime.h"
+#include "utime-helpers.h"
 #include "utime-messages.h"
 #include "utime-mem-types.h"
+#include <glusterfs/call-stub.h>
 
 int32_t
 gf_utime_invalidate(xlator_t *this, inode_t *inode)
@@ -133,6 +135,124 @@ mem_acct_init(xlator_t *this)
 }
 
 int32_t
+gf_utime_set_mdata_setxattr_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int op_ret, int op_errno,
+                                dict_t *xdata)
+{
+    /* Don't fail lookup if mdata setxattr fails */
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, UTIME_MSG_SET_MDATA_FAILED,
+               "dict set of key for set-ctime-mdata failed");
+    }
+    call_resume(frame->local);
+    return 0;
+}
+
+int32_t
+gf_utime_set_mdata_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *stbuf, dict_t *xdata,
+                              struct iatt *postparent)
+{
+    dict_t *dict = NULL;
+    struct mdata_iatt *mdata = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+
+    if (!op_ret && dict_get(xdata, GF_XATTR_MDATA_KEY) == NULL) {
+        dict = dict_new();
+        if (!dict) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        mdata = GF_MALLOC(sizeof(struct mdata_iatt), gf_common_mt_char);
+        if (mdata == NULL) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        iatt_to_mdata(mdata, stbuf);
+        ret = dict_set_mdata(dict, CTIME_MDATA_XDATA_KEY, mdata, _gf_false);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, UTIME_MSG_NO_MEMORY,
+                   "dict set of key for set-ctime-mdata failed");
+            goto err;
+        }
+        frame->local = fop_lookup_cbk_stub(frame, default_lookup_cbk, op_ret,
+                                           op_errno, inode, stbuf, xdata,
+                                           postparent);
+        if (!frame->local) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, UTIME_MSG_NO_MEMORY,
+                   "lookup_cbk stub allocation failed");
+            goto stub_err;
+        }
+
+        loc.inode = inode_ref(inode);
+        gf_uuid_copy(loc.gfid, stbuf->ia_gfid);
+        STACK_WIND(frame, gf_utime_set_mdata_setxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setxattr, &loc, dict, 0, NULL);
+
+        dict_unref(dict);
+        inode_unref(loc.inode);
+        return 0;
+    }
+
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf, xdata,
+                        postparent);
+    return 0;
+
+err:
+    if (mdata) {
+        GF_FREE(mdata);
+    }
+stub_err:
+    if (dict) {
+        dict_unref(dict);
+    }
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int
+gf_utime_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int op_errno = -1;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    xdata = xdata ? dict_ref(xdata) : dict_new();
+    if (!xdata) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, GF_XATTR_MDATA_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, UTIME_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GF_XATTR_MDATA_KEY);
+        op_errno = -ret;
+        goto free_dict;
+    }
+
+    STACK_WIND(frame, gf_utime_set_mdata_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    dict_unref(xdata);
+    return 0;
+
+free_dict:
+    dict_unref(xdata);
+err:
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
 init(xlator_t *this)
 {
     utime_priv_t *utime = NULL;
@@ -182,19 +302,27 @@ notify(xlator_t *this, int event, void *data, ...)
 }
 
 struct xlator_fops fops = {
-    /* TODO: Need to go through other fops and
-     *       check if they modify time attributes
-     */
-    .rename = gf_utime_rename,       .mknod = gf_utime_mknod,
-    .readv = gf_utime_readv,         .fremovexattr = gf_utime_fremovexattr,
-    .open = gf_utime_open,           .create = gf_utime_create,
-    .mkdir = gf_utime_mkdir,         .writev = gf_utime_writev,
-    .rmdir = gf_utime_rmdir,         .fallocate = gf_utime_fallocate,
-    .truncate = gf_utime_truncate,   .symlink = gf_utime_symlink,
-    .zerofill = gf_utime_zerofill,   .link = gf_utime_link,
-    .ftruncate = gf_utime_ftruncate, .unlink = gf_utime_unlink,
-    .setattr = gf_utime_setattr,     .fsetattr = gf_utime_fsetattr,
-    .opendir = gf_utime_opendir,     .removexattr = gf_utime_removexattr,
+    .rename = gf_utime_rename,
+    .mknod = gf_utime_mknod,
+    .readv = gf_utime_readv,
+    .fremovexattr = gf_utime_fremovexattr,
+    .open = gf_utime_open,
+    .create = gf_utime_create,
+    .mkdir = gf_utime_mkdir,
+    .writev = gf_utime_writev,
+    .rmdir = gf_utime_rmdir,
+    .fallocate = gf_utime_fallocate,
+    .truncate = gf_utime_truncate,
+    .symlink = gf_utime_symlink,
+    .zerofill = gf_utime_zerofill,
+    .link = gf_utime_link,
+    .ftruncate = gf_utime_ftruncate,
+    .unlink = gf_utime_unlink,
+    .setattr = gf_utime_setattr,
+    .fsetattr = gf_utime_fsetattr,
+    .opendir = gf_utime_opendir,
+    .removexattr = gf_utime_removexattr,
+    .lookup = gf_utime_lookup,
 };
 struct xlator_cbks cbks = {
     .invalidate = gf_utime_invalidate,
diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c
index 8c9637876ac..dd2b8e007fd 100644
--- a/xlators/storage/posix/src/posix-inode-fd-ops.c
+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c
@@ -2629,6 +2629,9 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
     gf_cs_obj_state state = -1;
     int i = 0;
     int len;
+    struct mdata_iatt mdata_iatt = {
+        0,
+    };
 
     DECLARE_OLD_FS_ID_VAR;
     SET_FS_ID(frame->root->uid, frame->root->gid);
@@ -2642,6 +2645,20 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
     priv = this->private;
     DISK_SPACE_CHECK_AND_GOTO(frame, priv, xdata, op_ret, op_errno, out);
 
+    ret = dict_get_mdata(dict, CTIME_MDATA_XDATA_KEY, &mdata_iatt);
+    if (ret == 0) {
+        /* This is initiated by lookup when ctime feature is enabled to create
+         * "trusted.glusterfs.mdata" xattr if not present. These are the files
+         * which were created when ctime feature is disabled.
+         */
+        ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, &mdata_iatt,
+                                                 &op_errno);
+        if (ret != 0) {
+            op_ret = -1;
+        }
+        goto out;
+    }
+
     MAKE_INODE_HANDLE(real_path, this, loc, NULL);
     if (!real_path) {
         op_ret = -1;
diff --git a/xlators/storage/posix/src/posix-messages.h b/xlators/storage/posix/src/posix-messages.h
index 32292756bc4..15e23ff7563 100644
--- a/xlators/storage/posix/src/posix-messages.h
+++ b/xlators/storage/posix/src/posix-messages.h
@@ -68,6 +68,7 @@ GLFS_MSGID(POSIX, P_MSG_XATTR_FAILED, P_MSG_NULL_GFID, P_MSG_FCNTL_FAILED,
            P_MSG_FALLOCATE_FAILED, P_MSG_STOREMDATA_FAILED,
            P_MSG_FETCHMDATA_FAILED, P_MSG_GETMDATA_FAILED,
            P_MSG_SETMDATA_FAILED, P_MSG_FRESHFILE, P_MSG_MUTEX_FAILED,
-           P_MSG_COPY_FILE_RANGE_FAILED, P_MSG_TIMER_DELETE_FAILED);
+           P_MSG_COPY_FILE_RANGE_FAILED, P_MSG_TIMER_DELETE_FAILED,
+           P_MSG_NOMEM);
 
 #endif /* !_GLUSTERD_MESSAGES_H_ */
diff --git a/xlators/storage/posix/src/posix-metadata.c b/xlators/storage/posix/src/posix-metadata.c
index 5a5e6cd744e..647c0bb3a65 100644
--- a/xlators/storage/posix/src/posix-metadata.c
+++ b/xlators/storage/posix/src/posix-metadata.c
@@ -245,6 +245,10 @@ __posix_get_mdata_xattr(xlator_t *this, const char *real_path, int _fd,
     if (ret == -1 || !mdata) {
         mdata = GF_CALLOC(1, sizeof(posix_mdata_t), gf_posix_mt_mdata_attr);
         if (!mdata) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, P_MSG_NOMEM,
+                   "Could not allocate mdata. file: %s: gfid: %s",
+                   real_path ? real_path : "null",
+                   inode ? uuid_utoa(inode->gfid) : "null");
             ret = -1;
             goto out;
         }
@@ -262,18 +266,8 @@ __posix_get_mdata_xattr(xlator_t *this, const char *real_path, int _fd,
             }
         } else {
             /* Failed to get mdata from disk, xattr missing.
-             * This happens on two cases.
-             * 1. File is created before ctime is enabled.
-             * 2. On new file creation.
-             *
-             * Do nothing, just return success. It is as
-             * good as ctime feature is not enabled for this
-             * file. For files created before ctime is enabled,
-             * time attributes gets updated into ctime structure
-             * once the metadata modification fop happens and
-             * time attributes become consistent eventually.
-             * For new files, it would obviously get updated
-             * before the fop completion.
+             * This happens when the file is created before
+             * ctime is enabled.
              */
             if (stbuf && op_errno != ENOENT) {
                 ret = 0;
@@ -345,6 +339,54 @@ posix_compare_timespec(struct timespec *first, struct timespec *second)
         return first->tv_sec - second->tv_sec;
 }
 
+int
+posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
+                                   struct mdata_iatt *mdata_iatt, int *op_errno)
+{
+    posix_mdata_t *mdata = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("posix", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    LOCK(&inode->lock);
+    {
+        mdata = GF_CALLOC(1, sizeof(posix_mdata_t), gf_posix_mt_mdata_attr);
+        if (!mdata) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, P_MSG_NOMEM,
+                   "Could not allocate mdata. gfid: %s",
+                   uuid_utoa(inode->gfid));
+            ret = -1;
+            *op_errno = ENOMEM;
+            goto unlock;
+        }
+
+        mdata->version = 1;
+        mdata->flags = 0;
+        mdata->ctime.tv_sec = mdata_iatt->ia_ctime;
+        mdata->ctime.tv_nsec = mdata_iatt->ia_ctime_nsec;
+        mdata->atime.tv_sec = mdata_iatt->ia_atime;
+        mdata->atime.tv_nsec = mdata_iatt->ia_atime_nsec;
+        mdata->mtime.tv_sec = mdata_iatt->ia_mtime;
+        mdata->mtime.tv_nsec = mdata_iatt->ia_mtime_nsec;
+
+        __inode_ctx_set1(inode, this, (uint64_t *)&mdata);
+
+        ret = posix_store_mdata_xattr(this, NULL, -1, inode, mdata);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_STOREMDATA_FAILED,
+                   "gfid: %s key:%s ", uuid_utoa(inode->gfid),
+                   GF_XATTR_MDATA_KEY);
+            *op_errno = errno;
+            goto unlock;
+        }
+    }
+unlock:
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
+
 /* posix_set_mdata_xattr updates the posix_mdata_t based on the flag
  * in inode context and stores it on disk
  */
@@ -372,6 +414,9 @@ posix_set_mdata_xattr(xlator_t *this, const char *real_path, int fd,
              */
             mdata = GF_CALLOC(1, sizeof(posix_mdata_t), gf_posix_mt_mdata_attr);
             if (!mdata) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, P_MSG_NOMEM,
+                       "Could not allocate mdata. file: %s: gfid: %s",
+                       real_path ? real_path : "null", uuid_utoa(inode->gfid));
                 ret = -1;
                 goto unlock;
             }
@@ -386,35 +431,11 @@ posix_set_mdata_xattr(xlator_t *this, const char *real_path, int fd,
                 __inode_ctx_set1(inode, this, (uint64_t *)&mdata);
             } else {
                 /*
-                 * This is the first time creating the time
-                 * attr. This happens when you activate this
-                 * feature, and the legacy file will not have
-                 * any xattr set.
-                 *
-                 * New files will create extended attributes.
-                 */
-
-                /*
-                 * TODO: This is wrong approach, because before
-                 * creating fresh xattr, we should consult
-                 * to all replica and/or distribution set.
-                 *
-                 * We should contact the time management
-                 * xlators, and ask them to create an xattr.
-                 */
-                /* We should not be relying on backend file's
-                 * time attributes to load the initial ctime
-                 * time attribute structure. This is incorrect
-                 * as each replica set would have witnessed the
-                 * file creation at different times.
-                 *
-                 * For new file creation, ctime, atime and mtime
-                 * should be same, hence initiate the ctime
-                 * structure with the time from the frame. But
-                 * for the files which were created before ctime
-                 * feature is enabled, this is not accurate but
-                 * still fine as the times would get eventually
-                 * accurate.
+                 * This is the first time creating the time attr. This happens
+                 * when you activate this feature. On this code path, only new
+                 * files will create mdata xattr. The legacy files (files
+                 * created before ctime enabled) will not have any xattr set.
+                 * The xattr on legacy file will be set via lookup.
                  */
 
                 /* Don't create xattr with utimes/utimensat, only update if
diff --git a/xlators/storage/posix/src/posix-metadata.h b/xlators/storage/posix/src/posix-metadata.h
index 3416148ea97..dc25e59c66a 100644
--- a/xlators/storage/posix/src/posix-metadata.h
+++ b/xlators/storage/posix/src/posix-metadata.h
@@ -53,5 +53,9 @@ posix_set_ctime_cfr(call_frame_t *frame, xlator_t *this,
                     const char *real_path_in, int fd_in, inode_t *inode_in,
                     struct iatt *stbuf_in, const char *read_path_put,
                     int fd_out, inode_t *inode_out, struct iatt *stbuf_out);
+int
+posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
+                                   struct mdata_iatt *mdata_iatt,
+                                   int *op_errno);
 
 #endif /* _POSIX_METADATA_H */