features/locks: Remove stale inodelk objects from 'blocked_locks' list

* In the event of a DISCONNECT from a client, as part of cleanup,
  inodelk objects are not removed from the blocked_locks list before
  being unref'd and freed, causing the brick process to crash at
  some point when the (now) stale object is accessed again in the list.

* Also during cleanup, it is pointless to try and grant lock to a
  previously blocked inodelk (say L1) as part of releasing another
  conflicting lock (L2), (which is a side-effect of L1 not being
  deleted from blocked_locks list before grant_blocked_inode_locks()
  in cleanup) if L1 is also associated with the DISCONNECTing client.
  This patch fixes the problem.

* Also, the codepath in cleanup of entrylks seems to be granting
  blocked inodelks, when it should be attempting to grant blocked
  entrylks, which is fixed in this patch.

Change-Id: I8493365c33020333b3f61aa15f505e4e7e6a9891
BUG: 1089470
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: http://review.gluster.org/7512
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

2 files changed, 52 insertions, 6 deletions

diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index dc86512bef0..4f5d8aca2c6 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -749,7 +749,7 @@ pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
 
 		dom = get_domain (pinode, l->volume);
 
-		grant_blocked_inode_locks (this, pinode, dom);
+		grant_blocked_entry_locks (this, pinode, dom);
 
 		pthread_mutex_lock (&pinode->mutex);
 		{
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index 2d9aa83eca4..d105e14f23d 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -404,7 +404,7 @@ pl_inodelk_log_cleanup (pl_inode_lock_t *lock)
 }
 
 
-/* Release all entrylks from this client */
+/* Release all inodelks from this client */
 int
 pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
 {
@@ -414,15 +414,16 @@ pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
         pl_inode_t *pl_inode = NULL;
 
         struct list_head released;
+        struct list_head unwind;
 
         INIT_LIST_HEAD (&released);
+        INIT_LIST_HEAD (&unwind);
 
 	pthread_mutex_lock (&ctx->lock);
         {
                 list_for_each_entry_safe (l, tmp, &ctx->inodelk_lockers,
 					  client_list) {
                         list_del_init (&l->client_list);
-			list_add_tail (&l->client_list, &released);
 
 			pl_inodelk_log_cleanup (l);
 
@@ -430,19 +431,64 @@ pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
 
 			pthread_mutex_lock (&pl_inode->mutex);
 			{
-				__delete_inode_lock (l);
+                        /* If the inodelk object is part of granted list but not
+                         * blocked list, then perform the following actions:
+                         * i.   delete the object from granted list;
+                         * ii.  grant other locks (from other clients) that may
+                         *      have been blocked on this inodelk; and
+                         * iii. unref the object.
+                         *
+                         * If the inodelk object (L1) is part of both granted
+                         * and blocked lists, then this means that a parallel
+                         * unlock on another inodelk (L2 say) may have 'granted'
+                         * L1 and added it to 'granted' list in
+                         * __grant_blocked_node_locks() (although using the
+                         * 'blocked_locks' member). In that case, the cleanup
+                         * codepath must try and grant other overlapping
+                         * blocked inodelks from other clients, now that L1 is
+                         * out of their way and then unref L1 in the end, and
+                         * leave it to the other thread (the one executing
+                         * unlock codepath) to unwind L1's frame, delete it from
+                         * blocked_locks list, and perform the last unref on L1.
+                         *
+                         * If the inodelk object (L1) is part of blocked list
+                         * only, the cleanup code path must:
+                         * i.   delete it from the blocked_locks list inside
+                         *      this critical section,
+                         * ii.  unwind its frame with EAGAIN,
+                         * iii. try and grant blocked inode locks from other
+                         *      clients that were otherwise grantable, but just
+                         *      got blocked to avoid leaving L1 to starve
+                         *      forever.
+                         * iv.  unref the object.
+                         */
+                                if (!list_empty (&l->list)) {
+                                        __delete_inode_lock (l);
+                                        list_add_tail (&l->client_list,
+                                                       &released);
+                                } else {
+                                        list_del_init(&l->blocked_locks);
+                                        list_add_tail (&l->client_list,
+                                                       &unwind);
+                                }
                         }
 			pthread_mutex_unlock (&pl_inode->mutex);
                 }
 	}
         pthread_mutex_unlock (&ctx->lock);
 
-        list_for_each_entry_safe (l, tmp, &released, client_list) {
+        list_for_each_entry_safe (l, tmp, &unwind, client_list) {
                 list_del_init (&l->client_list);
 
-		if (l->frame)
+                if (l->frame)
 			STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN,
 					     NULL);
+                list_add_tail (&l->client_list, &released);
+
+        }
+
+        list_for_each_entry_safe (l, tmp, &released, client_list) {
+                list_del_init (&l->client_list);
 
 		pl_inode = l->pl_inode;