features/quota: Improvements to quota

* Two stages of quota enforcement is done:

  Soft and hard quota Upon reaching soft quota limit on the directory
  it logs/alerts in the quota daemon log (ie DEFAULT_LOG_DIR/quotad.log)
  and no more writes allowed after hard
  quota limit. After reaching the soft-limit the daemon alerts the
  user/admin repeatively for every 'alert-time', which is
  configurable.

* Quota enforcer is moved to server-side.

  It  takes care of enforcing quota. Since enforcer doesn't have the
  cluster view, it relies on another service called
  quota-aggregator. Aggregator, on query can return the size of a
  directory based on the cluster view.

  Enforcer is always loaded in the server graph and is by passed if
  the feature is not enabled.

  Options specific to enforcer:

  server-quota - Specifies whether the feature is on/off. It is used
  to by pass the quota if turned off.

  deem-statfs - If set to on, it takes quota limits into consideration
  while estimating fs size. (df command). The algorithm followed is,
  i.   Adjust statvfs based on limit configured on root.
  ii.  If limit is set on the inode passed, use size/limits on that inode to
       populate statvfs. Otherwise, use size/limits configured on root.
  iii. Upon statvfs, update the ctx->size on the inode.
  iv.  Don't let DHT aggregate, instead take the maximum of the usages from the
       subvols of the DHT, since each of it contains the complete information.

  Enforcer also makes use of gfid-to-path conversion functionality to
  work correctly when a client like nfs predominently relies on
  nameless lookups.

* Quota Aggregator acts as a thin client to provide cluster view

  Its a lightweight *gluster client* process with no mount point,
  started upon enabling quota or restarting the volume. This is a
  single process run on each brick, which can answer queries on all
  volumes in the cluster. Its volfile stored in
  GLUSTERD_DEFAULT_WORKING_DIR/quotad/quotad.vol.

Credits:
Raghavendra Bhat        <rabhat@redhat.com>
Varun Shastry           <vshastry@redhat.com>
Shishir Gowda           <sgowda@redhat.com>
Kruthika Dhananjay      <kdhananj@redhat.com>
Brian Foster            <bfoster@redhat.com>
Krishnan Parthasarathi  <kparthas@redhat.com>

Change-Id: Id1cb25b414951da34c665a55f77385d482e0f9de
BUG: 969461
Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: http://review.gluster.org/5952
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

3 files changed, 118 insertions, 7 deletions

diff --git a/tests/basic/quota-nfs-anon.t b/tests/basic/quota-nfs-anon.t
new file mode 100644
index 00000000000..7b5ea5f28e0
--- /dev/null
+++ b/tests/basic/quota-nfs-anon.t
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1}
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST $CLI volume quota $V0 enable;
+
+## Mount NFS
+TEST mount -t nfs -o nolock,soft,intr $H0:/$V0 $N0;
+mkdir -p $N0/0/1
+TEST $CLI volume quota $V0 limit-usage /0/1 1GB 75%;
+
+deep=/0/1/2/3/4/5/6/7/8/9
+mkdir -p $N0/$deep
+dd if=/dev/zero of=$N0/$deep/file bs=1M count=502 &
+
+kill_brick $V0 $H0 $B0/${V0}{1}
+kill -TERM $(get_nfs_pid)
+
+$CLI volume start $V0 force;
+
+
+cleanup;
diff --git a/tests/basic/quota.t b/tests/basic/quota.t
index ef015a30d92..5c531adbf79 100755
--- a/tests/basic/quota.t
+++ b/tests/basic/quota.t
@@ -9,21 +9,37 @@ TEST glusterd
 TEST pidof glusterd
 TEST $CLI volume info;
 
-TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+TEST $CLI volume create $V0 replica 2  $H0:$B0/${V0}{1,2,3,4};
 
-function limit_on()
+function hard_limit()
 {
         local QUOTA_PATH=$1;
-        $CLI volume quota $V0 list | grep "$QUOTA_PATH" | awk '{print $2}'
+        $CLI volume quota $V0 list $QUOTA_PATH | grep "$QUOTA_PATH" | awk '{print $2}'
+}
+
+function soft_limit()
+{
+        local QUOTA_PATH=$1;
+        $CLI volume quota $V0 list $QUOTA_PATH | grep "$QUOTA_PATH" | awk '{print $3}'
+}
+
+function usage()
+{
+        local QUOTA_PATH=$1;
+        $CLI volume quota $V0 list $QUOTA_PATH | grep "$QUOTA_PATH" | awk '{print $4}'
 }
 
 EXPECT "$V0" volinfo_field $V0 'Volume Name';
 EXPECT 'Created' volinfo_field $V0 'Status';
-EXPECT '8' brick_count $V0
+EXPECT '4' brick_count $V0
 
 TEST $CLI volume start $V0;
 EXPECT 'Started' volinfo_field $V0 'Status';
 
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST mkdir -p $M0/test_dir/in_test_dir
+
 ## ------------------------------
 ## Verify quota commands
 ## ------------------------------
@@ -33,19 +49,66 @@ TEST $CLI volume quota $V0 limit-usage /test_dir 100MB
 
 TEST $CLI volume quota $V0 limit-usage /test_dir/in_test_dir 150MB
 
-EXPECT "150MB" limit_on "/test_dir/in_test_dir";
+EXPECT "150.0MB" hard_limit "/test_dir/in_test_dir";
+EXPECT "80%" soft_limit "/test_dir/in_test_dir";
 
 TEST $CLI volume quota $V0 remove /test_dir/in_test_dir
 
-EXPECT "100MB" limit_on "/test_dir";
+EXPECT "100.0MB" hard_limit "/test_dir";
+
+TEST $CLI volume quota $V0 limit-usage /test_dir 10MB
+EXPECT "10.0MB" hard_limit "/test_dir";
+EXPECT "80%" soft_limit "/test_dir";
+
+TEST $CLI volume quota $V0 soft-timeout 0
+TEST $CLI volume quota $V0 hard-timeout 0
 
-TEST $CLI volume quota $V0 disable
 ## ------------------------------
+## Verify quota enforcement
+## -----------------------------
+
+TEST ! dd if=/dev/urandom of=$M0/test_dir/1.txt bs=1M count=12
+TEST rm $M0/test_dir/1.txt
 
+# wait for marker's accounting to complete
+EXPECT_WITHIN 10 "0Bytes" usage "/test_dir"
+
+TEST dd if=/dev/urandom of=$M0/test_dir/2.txt bs=1M count=8
+EXPECT_WITHIN 20 "8.0MB" usage "/test_dir"
+TEST rm $M0/test_dir/2.txt
+EXPECT_WITHIN 10 "0Bytes" usage "/test_dir"
+
+## rename tests
+TEST dd if=/dev/urandom of=$M0/test_dir/2 bs=1M count=8
+EXPECT_WITHIN 20 "8.0MB" usage "/test_dir"
+TEST mv $M0/test_dir/2 $M0/test_dir/0
+EXPECT_WITHIN 10 "8.0MB" usage "/test_dir"
+TEST rm $M0/test_dir/0
+EXPECT_WITHIN 10 "0Bytes" usage "/test_dir"
+
+## ---------------------------
+
+## ------------------------------
+## Check if presence of nfs mount results in ESTALE errors for I/O
+#  on a fuse mount. Note: Quota command internally uses a fuse mount,
+#  though this may change.
+## -----------------------------
+
+TEST mount -t nfs -o nolock,soft,intr $H0:/$V0 $N0;
+TEST $CLI volume quota $V0 limit-usage /test_dir 100MB
+
+TEST $CLI volume quota $V0 limit-usage /test_dir/in_test_dir 150MB
+
+EXPECT "150.0MB" hard_limit "/test_dir/in_test_dir";
+## -----------------------------
+
+TEST $CLI volume quota $V0 disable
 TEST $CLI volume stop $V0;
 EXPECT 'Stopped' volinfo_field $V0 'Status';
 
 TEST $CLI volume delete $V0;
 TEST ! $CLI volume info $V0;
 
+umount -l $N0
+
 cleanup;
diff --git a/tests/include.rc b/tests/include.rc
index de28241c2ac..40dd8ac96b9 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -214,6 +214,8 @@ function _TEST_IN_LOOP()
 function cleanup()
 {
 	killall -15 glusterfs glusterfsd glusterd 2>/dev/null || true;
+        # allow completion of signal handlers for SIGTERM before issue SIGKILL
+        sleep 1
 	killall -9 glusterfs glusterfsd glusterd 2>/dev/null || true;
 
         MOUNTPOINTS=`mount | grep "$B0/" | awk '{print $3}'`