storage/posix: Use more granular mutex locks for atomic writes

Change-Id: I7a5167de77fabf19c5151775b553913a1af5a765
BUG: 1421938
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: https://review.gluster.org/16785
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra Bhat <raghavendra@redhat.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

3 files changed, 33 insertions, 8 deletions

diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index af3d01b469c..f619895ba4c 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -2194,10 +2194,12 @@ __posix_inode_ctx_get (inode_t *inode, xlator_t *this)
                 return NULL;
 
         pthread_mutex_init (&ctx_p->xattrop_lock, NULL);
+        pthread_mutex_init (&ctx_p->write_atomic_lock, NULL);
 
         ret = __inode_ctx_set (inode, this, (uint64_t *)&ctx_p);
         if (ret < 0) {
                 pthread_mutex_destroy (&ctx_p->xattrop_lock);
+                pthread_mutex_destroy (&ctx_p->write_atomic_lock);
                 GF_FREE (ctx_p);
                 return NULL;
         }
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 123fa9eb4df..f3fca45492b 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -137,6 +137,7 @@ posix_forget (xlator_t *this, inode_t *inode)
         }
 out:
         pthread_mutex_destroy (&ctx->xattrop_lock);
+        pthread_mutex_destroy (&ctx->write_atomic_lock);
         GF_FREE (ctx);
         return ret;
 }
@@ -686,6 +687,7 @@ posix_do_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
         int32_t             op_errno = 0;
         struct posix_fd    *pfd    = NULL;
         gf_boolean_t        locked = _gf_false;
+        posix_inode_ctx_t  *ctx    = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -701,9 +703,15 @@ posix_do_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto out;
         }
 
+        ret = posix_inode_ctx_get_all (fd->inode, this, &ctx);
+        if (ret < 0) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         if (dict_get (xdata, GLUSTERFS_WRITE_UPDATE_ATOMIC)) {
                 locked = _gf_true;
-                LOCK(&fd->inode->lock);
+                pthread_mutex_lock (&ctx->write_atomic_lock);
         }
 
         ret = posix_fdstat (this, pfd->fd, statpre);
@@ -730,7 +738,7 @@ posix_do_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 out:
         if (locked) {
-                UNLOCK (&fd->inode->lock);
+                pthread_mutex_unlock (&ctx->write_atomic_lock);
                 locked = _gf_false;
         }
         SET_TO_OLD_FS_ID ();
@@ -844,6 +852,7 @@ posix_do_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         int32_t            flags     = 0;
         struct posix_fd   *pfd       = NULL;
         gf_boolean_t       locked    = _gf_false;
+        posix_inode_ctx_t *ctx       = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -859,9 +868,15 @@ posix_do_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                 goto out;
         }
 
+        ret = posix_inode_ctx_get_all (fd->inode, this, &ctx);
+        if (ret < 0) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         if (dict_get (xdata, GLUSTERFS_WRITE_UPDATE_ATOMIC)) {
                 locked = _gf_true;
-                LOCK(&fd->inode->lock);
+                pthread_mutex_lock (&ctx->write_atomic_lock);
         }
 
         ret = posix_fdstat (this, pfd->fd, statpre);
@@ -911,7 +926,7 @@ fsync:
 
 out:
 	if (locked) {
-		UNLOCK (&fd->inode->lock);
+		pthread_mutex_unlock (&ctx->write_atomic_lock);
 		locked = _gf_false;
 	}
         SET_TO_OLD_FS_ID ();
@@ -3325,6 +3340,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	gf_boolean_t           locked = _gf_false;
 	gf_boolean_t           write_append = _gf_false;
 	gf_boolean_t           update_atomic = _gf_false;
+        posix_inode_ctx_t     *ctx      = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -3368,9 +3384,15 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
          * as of today).
          */
 
+        op_ret = posix_inode_ctx_get_all (fd->inode, this, &ctx);
+        if (op_ret < 0) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
         if (write_append || update_atomic) {
 		locked = _gf_true;
-		LOCK(&fd->inode->lock);
+		pthread_mutex_lock (&ctx->write_atomic_lock);
         }
 
         op_ret = posix_fdstat (this, _fd, &preop);
@@ -3390,7 +3412,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                                  (pfd->flags & O_DIRECT));
 
 	if (locked && (!update_atomic)) {
-		UNLOCK (&fd->inode->lock);
+		pthread_mutex_unlock (&ctx->write_atomic_lock);
 		locked = _gf_false;
 	}
 
@@ -3420,7 +3442,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         }
 
 	if (locked) {
-		UNLOCK (&fd->inode->lock);
+		pthread_mutex_unlock (&ctx->write_atomic_lock);
 		locked = _gf_false;
 	}
 
@@ -3446,7 +3468,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 out:
 
 	if (locked) {
-		UNLOCK (&fd->inode->lock);
+		pthread_mutex_unlock (&ctx->write_atomic_lock);
 		locked = _gf_false;
 	}
 
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index e6304250d14..01a0bfa6860 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -193,6 +193,7 @@ typedef struct {
 typedef struct {
         uint64_t unlink_flag;
         pthread_mutex_t xattrop_lock;
+        pthread_mutex_t write_atomic_lock;
 } posix_inode_ctx_t;
 
 #define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)