features/locks: Make inodelk ref-counted

Ref when the object is allocated.
Unref after the response is submitted to that inodelk.

Ref when the lock is granted.
Unref when the lock is unlocked.

Change-Id: I2bfe9182b67bea7dc7b9d0ed9f99f1c7fa0b8a3c
BUG: 783449
Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Reviewed-on: http://review.gluster.com/2891
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

5 files changed, 71 insertions, 47 deletions

diff --git a/xlators/features/locks/src/clear.c b/xlators/features/locks/src/clear.c
index ad6d0893cef..d64a15b09e4 100644
--- a/xlators/features/locks/src/clear.c
+++ b/xlators/features/locks/src/clear.c
@@ -223,7 +223,9 @@ clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
         int                     bcount          = 0;
         int                     gcount          = 0;
         gf_boolean_t            chk_range       = _gf_false;
+        struct list_head        released;
 
+        INIT_LIST_HEAD (&released);
         if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
                 *op_errno = EINVAL;
                 goto out;
@@ -247,17 +249,23 @@ blkd:
                                 continue;
 
                         bcount++;
-                        list_del_init (&ilock->list);
-                        pl_trace_out (this, ilock->frame, NULL, NULL, F_SETLKW,
-                                      &ilock->user_flock, -1, EAGAIN,
-                                      ilock->volume);
-                        STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
-                                             EAGAIN);
-                        GF_FREE (ilock);
+                        list_del_init (&ilock->blocked_locks);
+                        list_add (&ilock->blocked_locks, &released);
                 }
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
+        list_for_each_entry_safe (ilock, tmp, &released, blocked_locks) {
+                list_del_init (&ilock->blocked_locks);
+                pl_trace_out (this, ilock->frame, NULL, NULL, F_SETLKW,
+                              &ilock->user_flock, -1, EAGAIN,
+                              ilock->volume);
+                STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
+                                     EAGAIN);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (ilock);
+        }
+
         if (!(args->kind & CLRLK_GRANTED)) {
                 ret = 0;
                 goto out;
@@ -276,14 +284,20 @@ granted:
 
                         gcount++;
                         list_del_init (&ilock->list);
-                        GF_FREE (ilock);
+                        list_add (&ilock->list, &released);
                 }
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
-        grant_blocked_inode_locks (this, pl_inode, dom);
+        list_for_each_entry_safe (ilock, tmp, &released, list) {
+                list_del_init (&ilock->list);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (ilock);
+        }
+
         ret = 0;
 out:
+        grant_blocked_inode_locks (this, pl_inode, dom);
         *blkd    = bcount;
         *granted = gcount;
         return ret;
diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h
index a8e1d3d424c..fe5ebada9a8 100644
--- a/xlators/features/locks/src/common.h
+++ b/xlators/features/locks/src/common.h
@@ -79,7 +79,7 @@ void
 __delete_inode_lock (pl_inode_lock_t *lock);
 
 void
-__destroy_inode_lock (pl_inode_lock_t *lock);
+__pl_inodelk_unref (pl_inode_lock_t *lock);
 
 void
 grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index cc716033047..b8b5643e81a 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -33,20 +33,28 @@
 #include "locks.h"
 #include "common.h"
 
-void
+inline void
 __delete_inode_lock (pl_inode_lock_t *lock)
 {
         list_del (&lock->list);
 }
 
-void
-__destroy_inode_lock (pl_inode_lock_t *lock)
+static inline void
+__pl_inodelk_ref (pl_inode_lock_t *lock)
+{
+        lock->ref++;
+}
+
+inline void
+__pl_inodelk_unref (pl_inode_lock_t *lock)
 {
-        GF_FREE (lock);
+        lock->ref--;
+        if (!lock->ref)
+                GF_FREE (lock);
 }
 
 /* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't conflict */
-static int
+static inline int
 inodelk_type_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
         if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
@@ -245,6 +253,7 @@ __lock_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
 
                 goto out;
         }
+        __pl_inodelk_ref (lock);
         gettimeofday (&lock->granted_time, NULL);
         list_add (&lock->list, &dom->inodelk_list);
 
@@ -296,8 +305,6 @@ __inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
         __delete_inode_lock (conf);
         gf_log (this->name, GF_LOG_DEBUG,
                 " Matching lock found for unlock");
-        __destroy_inode_lock (lock);
-
 
 out:
         return conf;
@@ -340,11 +347,6 @@ grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *
 
         INIT_LIST_HEAD (&granted);
 
-        if (list_empty (&dom->blocked_inodelks)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No blocked locks to be granted for domain: %s", dom->domain);
-        }
-
         pthread_mutex_lock (&pl_inode->mutex);
         {
                 __grant_blocked_inode_locks (this, pl_inode, &granted, dom);
@@ -366,6 +368,14 @@ grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *
                 STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0);
         }
 
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
+                        list_del_init (&lock->blocked_locks);
+                        __pl_inodelk_unref (lock);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
 }
 
 /* Release all inodelks from this transport */
@@ -378,13 +388,11 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
 
         pl_inode_t * pinode = NULL;
 
-        struct list_head granted;
         struct list_head released;
 
         char *path = NULL;
         char *file = NULL;
 
-        INIT_LIST_HEAD (&granted);
         INIT_LIST_HEAD (&released);
 
         pinode = pl_inode_get (this, inode);
@@ -439,7 +447,7 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
                         }
 
                         __delete_inode_lock (l);
-                        __destroy_inode_lock (l);
+                        __pl_inodelk_unref (l);
                 }
         }
         if (path)
@@ -451,7 +459,8 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
                 list_del_init (&l->blocked_locks);
 
                 STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN);
-                GF_FREE (l);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (l);
         }
 
         grant_blocked_inode_locks (this, pinode, dom);
@@ -465,12 +474,13 @@ pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
 {
         int ret = -EINVAL;
         pl_inode_lock_t *retlock = NULL;
+        gf_boolean_t    unref = _gf_true;
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
                 if (lock->fl_type != F_UNLCK) {
                         ret = __lock_inodelk (this, pl_inode, lock, can_block, dom);
-                        if (ret == 0)
+                        if (ret == 0) {
                                 gf_log (this->name, GF_LOG_TRACE,
                                         "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
                                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
@@ -478,8 +488,7 @@ pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
                                         lkowner_utoa (&lock->owner),
                                         lock->fl_start,
                                         lock->fl_end);
-
-                        if (ret == -EAGAIN)
+                        } else if (ret == -EAGAIN) {
                                 gf_log (this->name, GF_LOG_TRACE,
                                         "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
                                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
@@ -487,25 +496,25 @@ pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
                                         lkowner_utoa (&lock->owner),
                                         lock->user_flock.l_start,
                                         lock->user_flock.l_len);
+                                if (can_block)
+                                        unref = _gf_false;
+                        }
+                } else {
+                        retlock = __inode_unlock_lock (this, lock, dom);
+                        if (!retlock) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "Bad Unlock issued on Inode lock");
+                                ret = -EINVAL;
+                                goto out;
+                        }
+                        __pl_inodelk_unref (retlock);
 
-                        goto out;
-                }
-
-
-                retlock = __inode_unlock_lock (this, lock, dom);
-                if (!retlock) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Bad Unlock issued on Inode lock");
-                        ret = -EINVAL;
-                        goto out;
+                        ret = 0;
                 }
-                __destroy_inode_lock (retlock);
-
-                ret = 0;
-
-
         }
 out:
+        if (unref)
+                __pl_inodelk_unref (lock);
         pthread_mutex_unlock (&pl_inode->mutex);
         grant_blocked_inode_locks (this, pl_inode, dom);
         return ret;
@@ -540,6 +549,7 @@ new_inode_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
 
         INIT_LIST_HEAD (&lock->list);
         INIT_LIST_HEAD (&lock->blocked_locks);
+        __pl_inodelk_ref (lock);
 
         return lock;
 }
@@ -629,7 +639,6 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                         }
                         gf_log (this->name, GF_LOG_TRACE, "returning EAGAIN");
                         op_errno = -ret;
-                        __destroy_inode_lock (reqlock);
                         goto unwind;
                 }
                 break;
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index 653cc4d6b8b..6df1235a2be 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -65,6 +65,7 @@ typedef struct __posix_lock posix_lock_t;
 struct __pl_inode_lock {
         struct list_head   list;
         struct list_head   blocked_locks; /* list_head pointing to blocked_inodelks */
+        int                ref;
 
         short              fl_type;
         off_t              fl_start;
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index 31522da245c..39991299c7f 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -1314,7 +1314,7 @@ pl_forget (xlator_t *this,
 
                                 list_for_each_entry_safe (ino_l, ino_tmp, &dom->inodelk_list, list) {
                                         __delete_inode_lock (ino_l);
-                                        __destroy_inode_lock (ino_l);
+                                        __pl_inodelk_unref (ino_l);
                                 }
 
                                 list_splice_init (&dom->blocked_inodelks, &inodelks_released);
@@ -1355,7 +1355,7 @@ pl_forget (xlator_t *this,
         list_for_each_entry_safe (ino_l, ino_tmp, &inodelks_released, blocked_locks) {
 
                 STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0);
-                __destroy_inode_lock (ino_l);
+                __pl_inodelk_unref (ino_l);
         }
 
         list_for_each_entry_safe (entry_l, entry_tmp, &entrylks_released, blocked_locks) {