features/quota: Improvements to quota

* Two stages of quota enforcement is done:

  Soft and hard quota Upon reaching soft quota limit on the directory
  it logs/alerts in the quota daemon log (ie DEFAULT_LOG_DIR/quotad.log)
  and no more writes allowed after hard
  quota limit. After reaching the soft-limit the daemon alerts the
  user/admin repeatively for every 'alert-time', which is
  configurable.

* Quota enforcer is moved to server-side.

  It  takes care of enforcing quota. Since enforcer doesn't have the
  cluster view, it relies on another service called
  quota-aggregator. Aggregator, on query can return the size of a
  directory based on the cluster view.

  Enforcer is always loaded in the server graph and is by passed if
  the feature is not enabled.

  Options specific to enforcer:

  server-quota - Specifies whether the feature is on/off. It is used
  to by pass the quota if turned off.

  deem-statfs - If set to on, it takes quota limits into consideration
  while estimating fs size. (df command). The algorithm followed is,
  i.   Adjust statvfs based on limit configured on root.
  ii.  If limit is set on the inode passed, use size/limits on that inode to
       populate statvfs. Otherwise, use size/limits configured on root.
  iii. Upon statvfs, update the ctx->size on the inode.
  iv.  Don't let DHT aggregate, instead take the maximum of the usages from the
       subvols of the DHT, since each of it contains the complete information.

  Enforcer also makes use of gfid-to-path conversion functionality to
  work correctly when a client like nfs predominently relies on
  nameless lookups.

* Quota Aggregator acts as a thin client to provide cluster view

  Its a lightweight *gluster client* process with no mount point,
  started upon enabling quota or restarting the volume. This is a
  single process run on each brick, which can answer queries on all
  volumes in the cluster. Its volfile stored in
  GLUSTERD_DEFAULT_WORKING_DIR/quotad/quotad.vol.

Credits:
Raghavendra Bhat        <rabhat@redhat.com>
Varun Shastry           <vshastry@redhat.com>
Shishir Gowda           <sgowda@redhat.com>
Kruthika Dhananjay      <kdhananj@redhat.com>
Brian Foster            <bfoster@redhat.com>
Krishnan Parthasarathi  <kparthas@redhat.com>

Change-Id: Id1cb25b414951da34c665a55f77385d482e0f9de
BUG: 969461
Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: http://review.gluster.org/5952
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

1 files changed, 97 insertions, 38 deletions

diff --git a/xlators/features/quota/src/quota.h b/xlators/features/quota/src/quota.h
index 84ecbb3084e..de522e6914f 100644
--- a/xlators/features/quota/src/quota.h
+++ b/xlators/features/quota/src/quota.h
@@ -12,20 +12,48 @@
 #include "config.h"
 #endif
 
+#ifndef _QUOTA_H
+#define _QUOTA_H
+
 #include "xlator.h"
 #include "call-stub.h"
 #include "defaults.h"
-#include "byte-order.h"
 #include "common-utils.h"
 #include "quota-mem-types.h"
+#include "glusterfs.h"
+#include "compat.h"
+#include "logging.h"
+#include "dict.h"
+#include "stack.h"
+#include "common-utils.h"
+#include "event.h"
+#include "globals.h"
+#include "rpcsvc.h"
+#include "rpc-clnt.h"
+#include "byte-order.h"
+#include "glusterfs3-xdr.h"
+#include "glusterfs3.h"
+#include "xdr-generic.h"
+#include "compat-errno.h"
+#include "protocol-common.h"
 
-#define QUOTA_XATTR_PREFIX      "trusted."
 #define DIRTY                   "dirty"
 #define SIZE                    "size"
 #define CONTRIBUTION            "contri"
 #define VAL_LENGTH              8
 #define READDIR_BUF             4096
 
+#ifndef UUID_CANONICAL_FORM_LEN
+#define UUID_CANONICAL_FORM_LEN 36
+#endif
+
+#define WIND_IF_QUOTAOFF(is_quota_on, label)     \
+        if (!is_quota_on)                       \
+                goto label;
+
+#define DID_REACH_LIMIT(lim, prev_size, cur_size)               \
+        ((cur_size) >= (lim) && (prev_size) < (lim))
+
 #define QUOTA_SAFE_INCREMENT(lock, var)         \
         do {                                    \
                 LOCK (lock);                    \
@@ -46,7 +74,7 @@
                                  gf_quota_mt_##type);   \
                 if (!var) {                             \
                         gf_log ("", GF_LOG_ERROR,       \
-                                "out of memory :(");    \
+                                "out of memory");    \
                         ret = -1;                       \
                         goto label;                     \
                 }                                       \
@@ -74,11 +102,17 @@
 #define GET_CONTRI_KEY(var, _vol_name, _gfid, _ret)             \
         do {                                                    \
                 char _gfid_unparsed[40];                        \
-                uuid_unparse (_gfid, _gfid_unparsed);           \
-                _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
-                                    "%s.%s." CONTRIBUTION,      \
-                                    _vol_name, _gfid_unparsed); \
-        } while (0)
+                if (_gfid != NULL) {                            \
+                        uuid_unparse (_gfid, _gfid_unparsed);   \
+                        _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
+                                            "%s.%s." CONTRIBUTION,      \
+                                            _vol_name, _gfid_unparsed); \
+                } else {                                                \
+                        _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
+                                            "%s.." CONTRIBUTION,       \
+                                            _vol_name);             \
+                }                                                       \
+         } while (0)
 
 
 #define GET_CONTRI_KEY_OR_GOTO(var, _vol_name, _gfid, label)    \
@@ -96,6 +130,8 @@
                         goto label;                             \
         } while (0)
 
+
+
 struct quota_dentry {
         char            *name;
         uuid_t           par;
@@ -105,47 +141,70 @@ typedef struct quota_dentry quota_dentry_t;
 
 struct quota_inode_ctx {
         int64_t          size;
-        int64_t          limit;
+        int64_t          hard_lim;
+        int64_t          soft_lim;
         struct iatt      buf;
         struct list_head parents;
         struct timeval   tv;
+        struct timeval   prev_log;
         gf_lock_t        lock;
 };
 typedef struct quota_inode_ctx quota_inode_ctx_t;
 
+struct quota_limit {
+        int64_t hard_lim;
+        int64_t soft_lim_percent;
+} __attribute__ ((packed));
+typedef struct quota_limit quota_limit_t;
+
 struct quota_local {
-        gf_lock_t    lock;
-        uint32_t     validate_count;
-        uint32_t     link_count;
-        loc_t        loc;
-        loc_t        oldloc;
-        loc_t        newloc;
-        loc_t        validate_loc;
-        int64_t      delta;
-        int32_t      op_ret;
-        int32_t      op_errno;
-        int64_t      size;
-        int64_t      limit;
-        char         just_validated;
-        inode_t     *inode;
-        call_stub_t *stub;
+        gf_lock_t           lock;
+        uint32_t            validate_count;
+        uint32_t            link_count;
+        loc_t               loc;
+        loc_t               oldloc;
+        loc_t               newloc;
+        loc_t               validate_loc;
+        int64_t             delta;
+        int32_t             op_ret;
+        int32_t             op_errno;
+        int64_t             size;
+        gf_boolean_t        skip_check;
+        char                just_validated;
+        fop_lookup_cbk_t    validate_cbk;
+        inode_t            *inode;
+        call_stub_t        *stub;
+        struct iobref      *iobref;
+        quota_limit_t       limit;
 };
-typedef struct quota_local quota_local_t;
+typedef struct quota_local  quota_local_t;
 
 struct quota_priv {
-        int64_t           timeout;
-        gf_boolean_t      consider_statfs;
-        struct list_head  limit_head;
-        gf_lock_t         lock;
+        uint32_t                soft_timeout;
+        uint32_t                hard_timeout;
+        uint32_t               log_timeout;
+        double                 default_soft_lim;
+        gf_boolean_t           is_quota_on;
+        gf_boolean_t           consider_statfs;
+        gf_lock_t              lock;
+        rpc_clnt_prog_t       *quota_enforcer;
+        struct rpcsvc_program *quotad_aggregator;
+        struct rpc_clnt       *rpc_clnt;
+        rpcsvc_t              *rpcsvc;
+        inode_table_t         *itable;
+        char                  *volume_uuid;
+        uint64_t               validation_count;
 };
-typedef struct quota_priv quota_priv_t;
+typedef struct quota_priv      quota_priv_t;
 
-struct limits {
-        struct list_head  limit_list;
-        char             *path;
-        int64_t           value;
-        uuid_t            gfid;
-};
-typedef struct limits     limits_t;
+int
+quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata, fop_lookup_cbk_t cbk);
+struct rpc_clnt *
+quota_enforcer_init (xlator_t *this, dict_t *options);
 
-uint64_t cn = 1;
+void
+quota_log_usage (xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
+                 int64_t delta);
+
+#endif