features/locks: Add lock revocation functionality to posix locks translator

Summary:
- Motivation: Prevents cluster instability by mis-behaving clients
  causing bricks to OOM due to inode/entry lock pile-ups.
- Adds option to strip clients of entry/inode locks after N seconds
- Adds option to clear ALL locks should the revocation threshold get hit
- Adds option to clear all or granted locks should the max-blocked
  threshold get hit (can be used in combination w/ revocation-clear-all).
- Options are:
    features.locks-revocation-secs <integer; 0 to disable>
    features.locks-revocation-clear-all [on/off]
    features.locks-revocation-max-blocked <integer>
- Adds monkey-locking option to ignore 1% of unlock requests (dev only)
    features.locks-monkey-unlocking [on/off]
- Adds logging to indicate revocation event & reason

Test Plan:
First you will need TWO fuse mounts for this repro.  Call them /mnt/patchy1 & /mnt/patchy2.

1. Enable monkey unlocking on the volume:
gluster vol set patchy features.locks-monkey-unlocking on

2. From the "patchy1", use DD or some other utility to begin writing to a file,
   eventually the dd will hang due to the dropped unlocked requests.  This now
   simulates the broken client.  Run:

for i in {1..1000};do dd if=/dev/zero of=/mnt/patchy1/testfile bs=1k count=10;done'

...this will eventually hang as the unlock request has been lost.

3. Goto another window and setup the mount "patchy2" @ /mnt/patchy2, and
   observe that 'echo "hello" >> /mnt/patchy2/testfile" will hang due to the
   inability of the client to take out the required lock.

4. Next, re-start the test this time enabling lock revocation; use a timeout of
   2-5 seconds for testing:
   'gluster vol set patchy features.locks-revocation-secs <2-5>'

5. Wait 2-5 seconds before executing step 3 above this time.  Observe that this
   time the access to the file will succeed, and the writes on patchy1 will
   unblock until they hit another failed unlock request due to
   "monkey-unlocking".

BUG: 1350867
Change-Id: I814b9f635fec53834a26db634d1300d9a61057d8
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/14816
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Krutika Dhananjay <kdhananj@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>

1 files changed, 118 insertions, 6 deletions

diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index 783c57e6381..4231d760cdc 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -16,9 +16,9 @@
 #include "list.h"
 
 #include "locks.h"
+#include "clear.h"
 #include "common.h"
 
-
 void
 __pl_entrylk_unref (pl_entry_lock_t *lock)
 {
@@ -111,6 +111,97 @@ __conflicting_entrylks (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
         return 0;
 }
 
+/* See comments in inodelk.c for details */
+static inline gf_boolean_t
+__stale_entrylk (xlator_t *this, pl_entry_lock_t *candidate_lock,
+                pl_entry_lock_t *requested_lock, time_t *lock_age_sec)
+{
+        posix_locks_private_t  *priv = NULL;
+        struct timeval curr;
+        gettimeofday (&curr, NULL);
+
+        priv = this->private;
+
+        /* Question: Should we just prune them all given the
+         * chance?  Or just the locks we are attempting to acquire?
+         */
+        if (names_conflict (candidate_lock->basename,
+                        requested_lock->basename)) {
+                *lock_age_sec = curr.tv_sec -
+                        candidate_lock->granted_time.tv_sec;
+                if (*lock_age_sec > priv->revocation_secs)
+                        return _gf_true;
+        }
+        return _gf_false;
+}
+
+/* See comments in inodelk.c for details */
+static gf_boolean_t
+__entrylk_prune_stale (xlator_t *this, pl_inode_t *pinode, pl_dom_list_t *dom,
+                 pl_entry_lock_t *lock)
+{
+        posix_locks_private_t  *priv = NULL;
+        pl_entry_lock_t *tmp = NULL;
+        pl_entry_lock_t *lk = NULL;
+        gf_boolean_t revoke_lock = _gf_false;
+        int bcount = 0;
+        int gcount = 0;
+        int op_errno = 0;
+        clrlk_args args;
+        args.opts = NULL;
+        time_t lk_age_sec = 0;
+        uint32_t max_blocked = 0;
+        char *reason_str = NULL;
+
+        priv = this->private;
+        args.type = CLRLK_ENTRY;
+        if (priv->revocation_clear_all == _gf_true)
+                args.kind = CLRLK_ALL;
+        else
+                args.kind = CLRLK_GRANTED;
+
+
+        if (list_empty (&dom->entrylk_list))
+                goto out;
+
+        pthread_mutex_lock (&pinode->mutex);
+        lock->pinode = pinode;
+        list_for_each_entry_safe (lk, tmp, &dom->entrylk_list, domain_list) {
+                if (__stale_entrylk (this, lk, lock, &lk_age_sec) == _gf_true) {
+                        revoke_lock = _gf_true;
+                        reason_str = "age";
+                        break;
+                }
+        }
+        max_blocked = priv->revocation_max_blocked;
+        if (max_blocked != 0 && revoke_lock == _gf_false) {
+                list_for_each_entry_safe (lk, tmp, &dom->blocked_entrylks,
+                                blocked_locks) {
+                        max_blocked--;
+                        if (max_blocked == 0) {
+                                revoke_lock = _gf_true;
+                                reason_str = "max blocked";
+                                break;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pinode->mutex);
+
+out:
+        if (revoke_lock == _gf_true) {
+                clrlk_clear_entrylk (this, pinode, dom, &args, &bcount, &gcount,
+                    &op_errno);
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Lock revocation [reason: %s; gfid: %s; domain: %s; "
+                        "age: %ld sec] - Entry lock revoked:  %d granted & %d "
+                        "blocked locks cleared", reason_str,
+                        uuid_utoa (pinode->gfid), dom->domain, lk_age_sec,
+                        gcount, bcount);
+        }
+
+        return revoke_lock;
+}
+
 /**
  * entrylk_grantable - is this lock grantable?
  * @inode: inode in which to look
@@ -546,6 +637,9 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
         pl_ctx_t        *ctx              =  NULL;
 	int              nonblock         =  0;
         gf_boolean_t     need_inode_unref =  _gf_false;
+        posix_locks_private_t  *priv = NULL;
+
+        priv = this->private;
 
         if (xdata)
                 dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
@@ -599,6 +693,24 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
          * current stack unwinds.
          */
         pinode->inode = inode_ref (inode);
+        if (priv->revocation_secs != 0) {
+                if (cmd != ENTRYLK_UNLOCK) {
+                        __entrylk_prune_stale (this, pinode, dom, reqlock);
+                } else if (priv->monkey_unlocking == _gf_true) {
+                        if (pl_does_monkey_want_stuck_lock ()) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                    "MONKEY LOCKING (forcing stuck lock)!");
+                                op_ret = 0;
+                                need_inode_unref = _gf_true;
+                                pthread_mutex_lock (&pinode->mutex);
+                                {
+                                        __pl_entrylk_unref (reqlock);
+                                }
+                                pthread_mutex_unlock (&pinode->mutex);
+                                goto out;
+                        }
+                }
+        }
 
         switch (cmd) {
         case ENTRYLK_LOCK_NB:
@@ -678,9 +790,6 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                         "a bug report at http://bugs.gluster.com", cmd);
                 goto out;
         }
-        if (need_inode_unref)
-                inode_unref (pinode->inode);
-
         /* The following (extra) unref corresponds to the ref that
          * was done at the time the lock was granted.
          */
@@ -689,6 +798,9 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
 
 out:
 
+        if (need_inode_unref)
+                inode_unref (pinode->inode);
+
         if (unwind) {
                 entrylk_trace_out (this, frame, volume, fd, loc, basename,
                                    cmd, type, op_ret, op_errno);
@@ -772,8 +884,6 @@ pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
         {
                 list_for_each_entry_safe (l, tmp, &ctx->entrylk_lockers,
 					  client_list) {
-                        list_del_init (&l->client_list);
-
 			pl_entrylk_log_cleanup (l);
 
 			pinode = l->pinode;
@@ -810,6 +920,8 @@ pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
                          *      blocked to avoid leaving L1 to starve forever.
                          * iv.  unref the object.
                          */
+                                list_del_init (&l->client_list);
+
                                 if (!list_empty (&l->domain_list)) {
                                         list_del_init (&l->domain_list);
                                         list_add_tail (&l->client_list,