storage/posix: Use more granular mutex locks for atomic writes

        Backport of: https://review.gluster.org/16785

Change-Id: I64aa561cb76ff9d4597d91fb5aeb64531698936a
BUG: 1427390
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: https://review.gluster.org/16892
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

3 files changed, 33 insertions, 8 deletions

diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 190298db235..0ed8015294b 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -2195,10 +2195,12 @@ __posix_inode_ctx_get (inode_t *inode, xlator_t *this)
                 return NULL;
 
         pthread_mutex_init (&ctx_p->xattrop_lock, NULL);
+        pthread_mutex_init (&ctx_p->write_atomic_lock, NULL);
 
         ret = __inode_ctx_set (inode, this, (uint64_t *)&ctx_p);
         if (ret < 0) {
                 pthread_mutex_destroy (&ctx_p->xattrop_lock);
+                pthread_mutex_destroy (&ctx_p->write_atomic_lock);
                 GF_FREE (ctx_p);
                 return NULL;
         }
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 6582d609af4..de995b980d0 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -136,6 +136,7 @@ posix_forget (xlator_t *this, inode_t *inode)
         }
 out:
         pthread_mutex_destroy (&ctx->xattrop_lock);
+        pthread_mutex_destroy (&ctx->write_atomic_lock);
         GF_FREE (ctx);
         return ret;
 }
@@ -685,6 +686,7 @@ posix_do_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
         int32_t             op_errno = 0;
         struct posix_fd    *pfd    = NULL;
         gf_boolean_t        locked = _gf_false;
+        posix_inode_ctx_t  *ctx    = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -700,9 +702,15 @@ posix_do_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto out;
         }
 
+        ret = posix_inode_ctx_get_all (fd->inode, this, &ctx);
+        if (ret < 0) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         if (dict_get (xdata, GLUSTERFS_WRITE_UPDATE_ATOMIC)) {
                 locked = _gf_true;
-                LOCK(&fd->inode->lock);
+                pthread_mutex_lock (&ctx->write_atomic_lock);
         }
 
         ret = posix_fdstat (this, pfd->fd, statpre);
@@ -729,7 +737,7 @@ posix_do_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 out:
         if (locked) {
-                UNLOCK (&fd->inode->lock);
+                pthread_mutex_unlock (&ctx->write_atomic_lock);
                 locked = _gf_false;
         }
         SET_TO_OLD_FS_ID ();
@@ -843,6 +851,7 @@ posix_do_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         int32_t            flags     = 0;
         struct posix_fd   *pfd       = NULL;
         gf_boolean_t       locked    = _gf_false;
+        posix_inode_ctx_t *ctx       = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -858,9 +867,15 @@ posix_do_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                 goto out;
         }
 
+        ret = posix_inode_ctx_get_all (fd->inode, this, &ctx);
+        if (ret < 0) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         if (dict_get (xdata, GLUSTERFS_WRITE_UPDATE_ATOMIC)) {
                 locked = _gf_true;
-                LOCK(&fd->inode->lock);
+                pthread_mutex_lock (&ctx->write_atomic_lock);
         }
 
         ret = posix_fdstat (this, pfd->fd, statpre);
@@ -910,7 +925,7 @@ fsync:
 
 out:
 	if (locked) {
-		UNLOCK (&fd->inode->lock);
+		pthread_mutex_unlock (&ctx->write_atomic_lock);
 		locked = _gf_false;
 	}
         SET_TO_OLD_FS_ID ();
@@ -3314,6 +3329,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	gf_boolean_t           locked = _gf_false;
 	gf_boolean_t           write_append = _gf_false;
 	gf_boolean_t           update_atomic = _gf_false;
+        posix_inode_ctx_t     *ctx      = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -3357,9 +3373,15 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
          * as of today).
          */
 
+        op_ret = posix_inode_ctx_get_all (fd->inode, this, &ctx);
+        if (op_ret < 0) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
         if (write_append || update_atomic) {
 		locked = _gf_true;
-		LOCK(&fd->inode->lock);
+		pthread_mutex_lock (&ctx->write_atomic_lock);
         }
 
         op_ret = posix_fdstat (this, _fd, &preop);
@@ -3379,7 +3401,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                                  (pfd->flags & O_DIRECT));
 
 	if (locked && (!update_atomic)) {
-		UNLOCK (&fd->inode->lock);
+		pthread_mutex_unlock (&ctx->write_atomic_lock);
 		locked = _gf_false;
 	}
 
@@ -3409,7 +3431,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         }
 
 	if (locked) {
-		UNLOCK (&fd->inode->lock);
+		pthread_mutex_unlock (&ctx->write_atomic_lock);
 		locked = _gf_false;
 	}
 
@@ -3435,7 +3457,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 out:
 
 	if (locked) {
-		UNLOCK (&fd->inode->lock);
+		pthread_mutex_unlock (&ctx->write_atomic_lock);
 		locked = _gf_false;
 	}
 
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index e6304250d14..01a0bfa6860 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -193,6 +193,7 @@ typedef struct {
 typedef struct {
         uint64_t unlink_flag;
         pthread_mutex_t xattrop_lock;
+        pthread_mutex_t write_atomic_lock;
 } posix_inode_ctx_t;
 
 #define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)