core, shard: Make shards inherit main file's O_DIRECT flag if present

If the application opens a file with O_DIRECT, the shards'
anon fds would also need to inherit the flag. Towards this,
shard xl would be passing the odirect flag in the @flags parameter
to the WRITEV fop. This will be used in anon fd resolution
and subsequent opening by posix xl.

Change-Id: Iddb75c9ed14ce5a8c5d2128ad09b749f46e3b0c2
BUG: 1342171
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: http://review.gluster.org/14191
Tested-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Smoke: Gluster Build System <jenkins@build.gluster.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

8 files changed, 66 insertions, 30 deletions

diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index 373f8da0f48..7177fa275db 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -744,7 +744,7 @@ fd_lookup_uint64 (inode_t *inode, uint64_t pid)
 }
 
 static fd_t *
-__fd_lookup_anonymous (inode_t *inode)
+__fd_lookup_anonymous (inode_t *inode, int32_t flags)
 {
         fd_t *iter_fd = NULL;
         fd_t *fd = NULL;
@@ -753,7 +753,7 @@ __fd_lookup_anonymous (inode_t *inode)
                 return NULL;
 
         list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                if (iter_fd->anonymous) {
+                if ((iter_fd->anonymous) && (flags == iter_fd->flags)) {
                         fd = __fd_ref (iter_fd);
                         break;
                 }
@@ -763,11 +763,11 @@ __fd_lookup_anonymous (inode_t *inode)
 }
 
 static fd_t *
-__fd_anonymous (inode_t *inode)
+__fd_anonymous (inode_t *inode, int32_t flags)
 {
         fd_t *fd = NULL;
 
-        fd = __fd_lookup_anonymous (inode);
+        fd = __fd_lookup_anonymous (inode, flags);
 
         /* if (fd); then we already have increased the refcount in
            __fd_lookup_anonymous(), so no need of one more fd_ref().
@@ -780,7 +780,7 @@ __fd_anonymous (inode_t *inode)
                         return NULL;
 
                 fd->anonymous = _gf_true;
-                fd->flags = GF_ANON_FD_FLAGS;
+                fd->flags = GF_ANON_FD_FLAGS|flags;
 
                 __fd_bind (fd);
 
@@ -798,7 +798,32 @@ fd_anonymous (inode_t *inode)
 
         LOCK (&inode->lock);
         {
-                fd = __fd_anonymous (inode);
+                fd = __fd_anonymous (inode, GF_ANON_FD_FLAGS);
+        }
+        UNLOCK (&inode->lock);
+
+        return fd;
+}
+
+fd_t *
+fd_anonymous_with_flags (inode_t *inode, int32_t flags)
+{
+        fd_t *fd = NULL;
+
+        LOCK (&inode->lock);
+        {
+                if (flags == 0)
+                        flags = GF_ANON_FD_FLAGS;
+                /* If this API is ever called with O_SYNC or O_DSYNC in @flags,
+                 * reset the bits associated with these flags before calling
+                 * __fd_anonymous(). That way, posix will do the open() without
+                 * these flags. And subsequently, posix_writev() (mostly) will
+                 * do the write within inode->lock on an fd without O_SYNC or
+                 * O_DSYNC and in its place to an fsync() outside of the locks
+                 * to simulate the effect of using these flags.
+                 */
+                flags &= (~(O_SYNC|O_DSYNC));
+                fd = __fd_anonymous (inode, flags);
         }
         UNLOCK (&inode->lock);
 
@@ -806,7 +831,7 @@ fd_anonymous (inode_t *inode)
 }
 
 fd_t*
-fd_lookup_anonymous (inode_t *inode)
+fd_lookup_anonymous (inode_t *inode, int32_t flags)
 {
         fd_t *fd = NULL;
 
@@ -818,7 +843,7 @@ fd_lookup_anonymous (inode_t *inode)
 
         LOCK (&inode->lock);
         {
-                fd = __fd_lookup_anonymous (inode);
+                fd = __fd_lookup_anonymous (inode, flags);
         }
         UNLOCK (&inode->lock);
         return fd;
diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
index a6dc48a0b0e..322cac7dd1f 100644
--- a/libglusterfs/src/fd.h
+++ b/libglusterfs/src/fd.h
@@ -136,11 +136,13 @@ fd_t *
 fd_lookup_uint64 (struct _inode *inode, uint64_t pid);
 
 fd_t*
-fd_lookup_anonymous (inode_t *inode);
+fd_lookup_anonymous (inode_t *inode, int32_t flags);
 
 fd_t *
 fd_anonymous (inode_t *inode);
 
+fd_t *
+fd_anonymous_with_flags (inode_t *inode, int32_t flags);
 
 gf_boolean_t
 fd_is_anonymous (fd_t *fd);
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 41418137ff0..41f36f4b2d8 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -2175,7 +2175,8 @@ afr_transaction_start (call_frame_t *frame, xlator_t *this)
         if (!local->transaction.eager_lock_on && local->loc.inode) {
                 fd = fd_lookup (local->loc.inode, frame->root->pid);
                 if (fd == NULL)
-                        fd = fd_lookup_anonymous (local->loc.inode);
+                        fd = fd_lookup_anonymous (local->loc.inode,
+                                                  GF_ANON_FD_FLAGS);
 
                 if (fd) {
                         afr_delayed_changelog_wake_up (this, fd);
diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c
index af2bdfbd6da..28f1a51d330 100644
--- a/xlators/features/shard/src/shard.c
+++ b/xlators/features/shard/src/shard.c
@@ -3638,6 +3638,7 @@ shard_common_inode_write_do (call_frame_t *frame, xlator_t *this)
         shard_local_t  *local             = NULL;
         struct iovec   *vec               = NULL;
         gf_boolean_t    wind_failed       = _gf_false;
+        gf_boolean_t    odirect           = _gf_false;
         off_t           orig_offset       = 0;
         off_t           shard_offset      = 0;
         off_t           vec_offset        = 0;
@@ -3668,6 +3669,9 @@ shard_common_inode_write_do (call_frame_t *frame, xlator_t *this)
                 return 0;
         }
 
+        if ((fd->flags & O_DIRECT) && (local->fop == GF_FOP_WRITE))
+                odirect = _gf_true;
+
         while (cur_block <= last_block) {
                 if (wind_failed) {
                         shard_common_inode_write_do_cbk (frame,
@@ -3725,6 +3729,13 @@ shard_common_inode_write_do (call_frame_t *frame, xlator_t *this)
                                                                  NULL, NULL);
                                 goto next;
                         }
+
+                        if (local->fop == GF_FOP_WRITE) {
+                                if (odirect)
+                                        local->flags = O_DIRECT;
+                                else
+                                        local->flags = GF_ANON_FD_FLAGS;
+                        }
                 }
 
                 shard_common_inode_write_wind (frame, this, anon_fd,
diff --git a/xlators/nfs/server/src/nfs-fops.c b/xlators/nfs/server/src/nfs-fops.c
index 52fdd9796c3..f6361f02161 100644
--- a/xlators/nfs/server/src/nfs-fops.c
+++ b/xlators/nfs/server/src/nfs-fops.c
@@ -1382,6 +1382,8 @@ nfs_fop_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         call_frame_t            *frame = NULL;
         int                     ret = -EFAULT;
         struct nfs_fop_local    *nfl = NULL;
+        int flags = 0;
+        nfs3_call_state_t       *cs = local;
 
         if ((!nfsx) || (!xl) || (!fd) || (!vector) || (!nfu) || (!srciobref))
                 return ret;
@@ -1399,8 +1401,20 @@ nfs_fop_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
 
         iobref_add (nfl->iobref, srciob);
 */
+
+        switch (cs->writetype) {
+        case UNSTABLE:
+                break;
+        case DATA_SYNC:
+                flags |= O_DSYNC;
+                break;
+        case FILE_SYNC:
+                flags |= O_SYNC;
+                break;
+        }
+
         STACK_WIND_COOKIE (frame, nfs_fop_writev_cbk, xl, xl,xl->fops->writev,
-                           fd, vector, count, offset, fd->flags, srciobref, NULL);
+                           fd, vector, count, offset, flags, srciobref, NULL);
         ret = 0;
 err:
         if (ret < 0) {
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index 8cf4db8174a..67b619ece13 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -2240,24 +2240,6 @@ nfs3_write_resume (void *carg)
 
         cs->fd = fd;    /* Gets unrefd when the call state is wiped. */
 
-/*
-  enum stable_how {
-  UNSTABLE = 0,
-  DATA_SYNC = 1,
-  FILE_SYNC = 2,
-  };
-*/
-	switch (cs->writetype) {
-	case UNSTABLE:
-		break;
-	case DATA_SYNC:
-		fd->flags |= O_DSYNC;
-		break;
-	case FILE_SYNC:
-		fd->flags |= O_SYNC;
-		break;
-	}
-
         ret = __nfs3_write_resume (cs);
         if (ret < 0)
                 stat = nfs3_errno_to_nfsstat3 (-ret);
diff --git a/xlators/protocol/server/src/server-resolve.c b/xlators/protocol/server/src/server-resolve.c
index 3257c6c701c..3e1411250b8 100644
--- a/xlators/protocol/server/src/server-resolve.c
+++ b/xlators/protocol/server/src/server-resolve.c
@@ -453,7 +453,7 @@ resolve_anonfd_simple (call_frame_t *frame)
 
         ret = 0;
 
-        state->fd = fd_anonymous (inode);
+        state->fd = fd_anonymous_with_flags (inode, state->flags);
 out:
         if (inode)
                 inode_unref (inode);
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 7a5a241fbb1..d953db249a0 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1686,6 +1686,7 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
 
         pfd->fd = _fd;
         pfd->dir = dir;
+        pfd->flags = fd->flags;
 
         ret = __fd_ctx_set (fd, this, (uint64_t) (long) pfd);
         if (ret != 0) {