storage/posix: Use granular mutex locks for pgfid update syscalls

Change-Id: Ie5d635951c483d858dc4be2a90fb24b8b5f4f02d
BUG: 1421938
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: https://review.gluster.org/16869
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

3 files changed, 54 insertions, 11 deletions

diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index f619895ba4c..cc862c259ca 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -2195,11 +2195,13 @@ __posix_inode_ctx_get (inode_t *inode, xlator_t *this)
 
         pthread_mutex_init (&ctx_p->xattrop_lock, NULL);
         pthread_mutex_init (&ctx_p->write_atomic_lock, NULL);
+        pthread_mutex_init (&ctx_p->pgfid_lock, NULL);
 
         ret = __inode_ctx_set (inode, this, (uint64_t *)&ctx_p);
         if (ret < 0) {
                 pthread_mutex_destroy (&ctx_p->xattrop_lock);
                 pthread_mutex_destroy (&ctx_p->write_atomic_lock);
+                pthread_mutex_destroy (&ctx_p->pgfid_lock);
                 GF_FREE (ctx_p);
                 return NULL;
         }
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 04d141f41b2..7c05c8f1929 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -157,6 +157,7 @@ posix_forget (xlator_t *this, inode_t *inode)
 out:
         pthread_mutex_destroy (&ctx->xattrop_lock);
         pthread_mutex_destroy (&ctx->write_atomic_lock);
+        pthread_mutex_destroy (&ctx->pgfid_lock);
         GF_FREE (ctx);
         return ret;
 }
@@ -179,6 +180,7 @@ posix_lookup (call_frame_t *frame, xlator_t *this,
         char        *pgfid_xattr_key   = NULL;
         int32_t     nlink_samepgfid    = 0;
         struct  posix_private *priv    = NULL;
+        posix_inode_ctx_t *ctx         = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -238,7 +240,14 @@ posix_lookup (call_frame_t *frame, xlator_t *this,
                                               PGFID_XATTR_KEY_PREFIX,
                                               loc->pargfid);
 
-                        LOCK (&loc->inode->lock);
+                        op_ret = posix_inode_ctx_get_all (loc->inode, this,
+                                                          &ctx);
+                        if (op_ret < 0) {
+                                op_errno = ENOMEM;
+                                goto out;
+                        }
+
+                        pthread_mutex_lock (&ctx->pgfid_lock);
                         {
                                 SET_PGFID_XATTR_IF_ABSENT (real_path,
                                                            pgfid_xattr_key,
@@ -247,7 +256,7 @@ posix_lookup (call_frame_t *frame, xlator_t *this,
                                                            this, unlock);
                         }
 unlock:
-                        UNLOCK (&loc->inode->lock);
+                        pthread_mutex_unlock (&ctx->pgfid_lock);
                 }
         }
 
@@ -1959,6 +1968,7 @@ posix_unlink (call_frame_t *frame, xlator_t *this,
         char                   uuid_str[GF_UUID_BUF_SIZE] = {0};
         char                   gfid_str[GF_UUID_BUF_SIZE] = {0};
         gf_boolean_t           get_link_count     = _gf_false;
+        posix_inode_ctx_t     *ctx                = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -2062,14 +2072,19 @@ posix_unlink (call_frame_t *frame, xlator_t *this,
         if (priv->update_pgfid_nlinks && (stbuf.ia_nlink > 1)) {
                 MAKE_PGFID_XATTR_KEY (pgfid_xattr_key, PGFID_XATTR_KEY_PREFIX,
                                       loc->pargfid);
-                LOCK (&loc->inode->lock);
+                op_ret = posix_inode_ctx_get_all (loc->inode, this, &ctx);
+                if (op_ret < 0) {
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+                pthread_mutex_lock (&ctx->pgfid_lock);
                 {
                         UNLINK_MODIFY_PGFID_XATTR (real_path, pgfid_xattr_key,
                                                    nlink_samepgfid, 0, op_ret,
                                                    this, unlock);
                 }
         unlock:
-                UNLOCK (&loc->inode->lock);
+                pthread_mutex_unlock (&ctx->pgfid_lock);
 
                 if (op_ret < 0) {
                         gf_msg (this->name, GF_LOG_WARNING, 0,
@@ -2423,6 +2438,8 @@ posix_rename (call_frame_t *frame, xlator_t *this,
         dict_t               *unwind_dict     = NULL;
         gf_boolean_t          locked          = _gf_false;
         gf_boolean_t          get_link_count  = _gf_false;
+        posix_inode_ctx_t    *ctx_old         = NULL;
+        posix_inode_ctx_t    *ctx_new         = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -2505,10 +2522,26 @@ posix_rename (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+        op_ret = posix_inode_ctx_get_all (oldloc->inode, this, &ctx_old);
+        if (op_ret < 0) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        if (newloc->inode) {
+                op_ret = posix_inode_ctx_get_all (newloc->inode, this, &ctx_new);
+                if (op_ret < 0) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        }
+
         if (IA_ISDIR (oldloc->inode->ia_type))
                 posix_handle_unset (this, oldloc->inode->gfid, NULL);
 
-        LOCK (&oldloc->inode->lock);
+        pthread_mutex_lock (&ctx_old->pgfid_lock);
         {
                 if (!IA_ISDIR (oldloc->inode->ia_type)
                     && priv->update_pgfid_nlinks) {
@@ -2524,7 +2557,7 @@ posix_rename (call_frame_t *frame, xlator_t *this,
 
                 if ((xdata) && (dict_get (xdata, GET_LINK_COUNT))
                     && (real_newpath) && (was_present)) {
-                        LOCK (&newloc->inode->lock);
+                        pthread_mutex_lock (&ctx_new->pgfid_lock);
                         locked = _gf_true;
                         get_link_count = _gf_true;
                         op_ret = posix_pstat (this, newloc->gfid, real_newpath,
@@ -2566,7 +2599,7 @@ posix_rename (call_frame_t *frame, xlator_t *this,
                 }
 
                 if (locked) {
-                        UNLOCK (&newloc->inode->lock);
+                        pthread_mutex_unlock (&ctx_new->pgfid_lock);
                         locked = _gf_false;
                 }
 
@@ -2591,10 +2624,10 @@ posix_rename (call_frame_t *frame, xlator_t *this,
         }
 unlock:
         if (locked) {
-                UNLOCK (&newloc->inode->lock);
+                pthread_mutex_unlock (&ctx_new->pgfid_lock);
                 locked = _gf_false;
         }
-        UNLOCK (&oldloc->inode->lock);
+        pthread_mutex_unlock (&ctx_old->pgfid_lock);
 
         if (op_ret < 0) {
                 gf_msg (this->name, GF_LOG_WARNING, 0, P_MSG_XATTR_FAILED,
@@ -2675,6 +2708,7 @@ posix_link (call_frame_t *frame, xlator_t *this,
         int32_t               nlink_samepgfid = 0;
         char                 *pgfid_xattr_key = NULL;
         gf_boolean_t          entry_created   = _gf_false;
+        posix_inode_ctx_t    *ctx             = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -2741,14 +2775,20 @@ posix_link (call_frame_t *frame, xlator_t *this,
                 MAKE_PGFID_XATTR_KEY (pgfid_xattr_key, PGFID_XATTR_KEY_PREFIX,
                                       newloc->pargfid);
 
-                LOCK (&newloc->inode->lock);
+                op_ret = posix_inode_ctx_get_all (newloc->inode, this, &ctx);
+                if (op_ret < 0) {
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+
+                pthread_mutex_lock (&ctx->pgfid_lock);
                 {
                         LINK_MODIFY_PGFID_XATTR (real_newpath, pgfid_xattr_key,
                                                  nlink_samepgfid, 0, op_ret,
                                                  this, unlock);
                 }
         unlock:
-                UNLOCK (&newloc->inode->lock);
+                pthread_mutex_unlock (&ctx->pgfid_lock);
 
                 if (op_ret < 0) {
                         gf_msg (this->name, GF_LOG_WARNING, 0,
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index 01a0bfa6860..4e43b1f2d32 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -194,6 +194,7 @@ typedef struct {
         uint64_t unlink_flag;
         pthread_mutex_t xattrop_lock;
         pthread_mutex_t write_atomic_lock;
+        pthread_mutex_t pgfid_lock;
 } posix_inode_ctx_t;
 
 #define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)