features/quota: Read the limit from the xattrs

Old approach:
This is based on the path. So, the config file stored as limit-set key value
pair in the volfile was the source of truth for any directory to be enforced
the quota limit. So, we were storing it in a list and everypath is compared
with the list. This hinders scaling with the order of kilos of quota
directories. This approach is different from the XFS where the below method of
inode based limit is employed this adds the admins to learn new stuff.

New approach:
This is based on the inode. After the path is set with the quota limit. The
limit sticks with the inode. So, we need not to store the paths in the xlator
context/options, instead get the xattrs in the lookup. Set the hard and soft
limits of context from there.

The additional change in the glusterd to set the xattrs when limit-usage
command issued will be submitted as a separate patch.

This patch also does:
* Move the logging from the quotad to quota-enforcer.
* Save the soft limit in the context.
* Some clean up in quota-enforcer

Change-Id: I2840f9377a4a1f76630741550873fe74758fa100
Signed-off-by: Varun Shastry <vshastry@redhat.com>

1 files changed, 154 insertions, 285 deletions

diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index 9ad7f2b8..c15eded4 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -293,6 +293,13 @@ int32_t
 quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
                    char *name, uuid_t par)
 {
+/* This macro is of limited scope, reuse it with care */
+/* crossed: (soft limit && hard timeout) or (soft timeout) */
+#define QUOTA_NEED_VALIDATION(ctx, priv)                                 \
+        ((usage > ctx->soft_lim &&                                       \
+          quota_timeout (&ctx->tv, priv->hard_timeout)) ||               \
+         (quota_timeout (&ctx->tv, priv->soft_timeout)))
+
         int32_t               ret            = -1;
         inode_t              *_inode         = NULL, *parent = NULL;
         quota_inode_ctx_t    *ctx            = NULL;
@@ -305,6 +312,7 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
         uint64_t              value          = 0;
         char                  just_validated = 0;
         uuid_t                trav_uuid      = {0,};
+        int64_t               usage          = 0;
 
 
         GF_VALIDATE_OR_GOTO ("quota", this, out);
@@ -341,20 +349,18 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
         }
 
         do {
-                if (ctx != NULL) {
+                if (ctx != NULL && ctx->hard_lim >= 0) {
                         LOCK (&ctx->lock);
                         {
-                                if (ctx->limit >= 0) {
-                                        if (!just_validated
-                                            && quota_timeout (&ctx->tv,
-                                                              priv->timeout)) {
+                                usage = ctx->size + delta;
+                                if (!just_validated) {
+                                        if (QUOTA_NEED_VALIDATION (ctx, priv)) {
                                                 need_validate = 1;
-                                        } else if ((ctx->size + delta)
-                                                   >= ctx->limit) {
-                                                local->op_ret = -1;
-                                                local->op_errno = EDQUOT;
-                                                need_unwind = 1;
                                         }
+                                } else if (usage >= ctx->hard_lim) {
+                                        local->op_ret = -1;
+                                        local->op_errno = EDQUOT;
+                                        need_unwind = 1;
                                 }
                         }
                         UNLOCK (&ctx->lock);
@@ -473,7 +479,7 @@ __quota_init_inode_ctx (inode_t *inode, int64_t limit, xlator_t *this,
 
         QUOTA_ALLOC_OR_GOTO (ctx, quota_inode_ctx_t, out);
 
-        ctx->limit = limit;
+        ctx->hard_lim = limit;
         if (buf)
                 ctx->buf = *buf;
 
@@ -534,43 +540,35 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
 {
+        struct xattr_limit_set {
+                int64_t         hard_lim;
+                int64_t         soft_lim;
+        }                 *read_xattr = NULL;
         int32_t            ret        = -1;
         char               found      = 0;
         quota_local_t     *local      = NULL;
         quota_inode_ctx_t *ctx        = NULL;
         quota_dentry_t    *dentry     = NULL;
-        int64_t           *size       = 0;
-        uint64_t           value      = 0;
-        limits_t          *limit_node = NULL;
+        int64_t            soft_lim   = 0;
+        int64_t            hard_lim   = -1;
         quota_priv_t      *priv       = NULL;
 
         local = frame->local;
         priv = this->private;
 
-        inode_ctx_get (inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+        GF_VALIDATE_OR_GOTO (this->name, (!op_ret), unwind);
+        GF_VALIDATE_OR_GOTO (this->name, dict, unwind);
+        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
 
-        if ((op_ret < 0) || (local == NULL)
-            || (((ctx == NULL) || (ctx->limit == local->limit))
-                && (local->limit < 0) && !((IA_ISREG (buf->ia_type))
-                                           || (IA_ISLNK (buf->ia_type))))) {
+        ret = dict_get_bin (dict, QUOTA_LIMIT_KEY, (void **) &read_xattr);
+        if (ret && !(IA_ISREG (buf->ia_type) || IA_ISLNK (buf->ia_type)))
                 goto unwind;
-        }
 
-        LOCK (&priv->lock);
-        {
-                list_for_each_entry (limit_node, &priv->limit_head,
-                                     limit_list) {
-                        if (strcmp (local->loc.path, limit_node->path) == 0) {
-                                uuid_copy (limit_node->gfid, buf->ia_gfid);
-                                break;
-                        }
-                }
-        }
-        UNLOCK (&priv->lock);
+        if (read_xattr)
+                hard_lim = read_xattr->hard_lim;
 
-        ret = quota_inode_ctx_get (local->loc.inode, local->limit, this, dict,
-                                   buf, &ctx, 1);
+        ret = quota_inode_ctx_get (local->loc.inode, hard_lim, this, dict, buf,
+                                   &ctx, 1);
         if ((ret == -1) || (ctx == NULL)) {
                 gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
                         "context in inode(gfid:%s)",
@@ -583,17 +581,10 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         LOCK (&ctx->lock);
         {
 
-                if (dict != NULL) {
-                        ret = dict_get_bin (dict, QUOTA_SIZE_KEY,
-                                            (void **) &size);
-                        if (ret == 0) {
-                                ctx->size = ntoh64 (*size);
-                                gettimeofday (&ctx->tv, NULL);
-                        }
-                }
-
-                if (local->limit != ctx->limit) {
-                        ctx->limit = local->limit;
+                if (hard_lim != -1) {
+                        soft_lim = read_xattr->soft_lim?
+                                   read_xattr->soft_lim: priv->default_soft_lim;
+                        ctx->soft_lim = (int64_t)ctx->hard_lim * (soft_lim/100);
                 }
 
                 ctx->buf = *buf;
@@ -647,8 +638,6 @@ quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
               dict_t *xattr_req)
 {
         int32_t             ret         = -1;
-        int64_t             limit       = -1;
-        limits_t           *limit_node  = NULL;
         gf_boolean_t        dict_newed  = _gf_false;
         quota_priv_t       *priv        = NULL;
         quota_local_t      *local       = NULL;
@@ -658,12 +647,6 @@ quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         QUOTA_WIND_IF_DISABLED (priv->quota_on, frame, this, lookup, loc,
                                 xattr_req);
 
-        list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
-                if (strcmp (limit_node->path, loc->path) == 0) {
-                        limit = limit_node->value;
-                }
-        }
-
         local = quota_local_new ();
         if (local == NULL) {
                 goto err;
@@ -676,12 +659,6 @@ quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         frame->local = local;
 
-        local->limit = limit;
-
-        if (limit < 0) {
-                goto wind;
-        }
-
         if (xattr_req == NULL) {
                 xattr_req  = dict_new ();
                 dict_newed = _gf_true;
@@ -692,7 +669,10 @@ quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto err;
         }
 
-wind:
+        ret = dict_set_uint64 (xattr_req, QUOTA_LIMIT_KEY, 0);
+        if (0 > ret)
+                goto err;
+
         STACK_WIND (frame, quota_lookup_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
 
@@ -711,6 +691,59 @@ err:
         return 0;
 }
 
+/* Logs if
+*  i.   Usage crossed soft limit
+*  ii.  Usage above soft limit and alert-time timed out
+*/
+void
+quota_log_usage (xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
+                 int64_t delta)
+{
+        struct timeval           cur_time       = {0,};
+        char                    *usage_str      = NULL;
+        char                    *path           = NULL;
+        int64_t                  cur_size       = 0;
+        quota_priv_t            *priv           = NULL;
+
+        priv = this->private;
+        cur_size = ctx->size + delta;
+        gettimeofday (&cur_time, NULL);
+
+        /* Usage crossed/reached soft limit? */
+        if (DID_REACH_LIMIT (ctx->soft_lim, ctx->size, cur_size)) {
+                ctx->prev_log = cur_time;
+                usage_str = gf_uint64_2human_readable (ctx->soft_lim);
+                inode_path (inode, NULL, &path);
+
+                gf_log (this->name, GF_LOG_ALERT, "Usage crossed soft limit: "
+                        "%s for %s", usage_str,
+                        path? path: uuid_utoa (inode->gfid));
+        }
+        /* Usage crossed/reached hard limit? */
+        else if (DID_REACH_LIMIT (ctx->hard_lim, ctx->size, cur_size)) {
+                ctx->prev_log = cur_time;
+                usage_str = gf_uint64_2human_readable (ctx->hard_lim);
+                inode_path (inode, NULL, &path);
+
+                gf_log (this->name, GF_LOG_ALERT, "Usage reached hard limit: "
+                        "%s for %s", usage_str,
+                        path? path: uuid_utoa (inode->gfid));
+        }
+        /* Usage is above soft limit and 'alert-time' timed out */
+        else if (cur_size > ctx->soft_lim &&
+                       quota_timeout (&ctx->prev_log, priv->log_timeout)) {
+                ctx->prev_log = cur_time;
+                usage_str = gf_uint64_2human_readable (cur_size);
+                inode_path (inode, NULL, &path);
+
+                gf_log (this->name, GF_LOG_ALERT, "Usage %s %s limit for %s ",
+                        usage_str, (cur_size < ctx->hard_lim)?
+                        "is above soft": "has reached hard",
+                        path? path: uuid_utoa (inode->gfid));
+        }
+        if (usage_str)
+                GF_FREE (usage_str);
+}
 
 void
 quota_update_size (xlator_t *this, inode_t *inode, char *name, uuid_t par,
@@ -735,7 +768,9 @@ quota_update_size (xlator_t *this, inode_t *inode, char *name, uuid_t par,
         }
 
         do {
-                if ((ctx != NULL) && (ctx->limit >= 0)) {
+                if ((ctx != NULL) && (ctx->hard_lim >= 0)) {
+                        quota_log_usage (this, ctx, _inode, delta);
+
                         LOCK (&ctx->lock);
                         {
                                 ctx->size += delta;
@@ -2029,7 +2064,7 @@ quota_send_dir_limit_to_cli (call_frame_t *frame, xlator_t *this,
                 goto out;
 
         ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        snprintf (dir_limit, 1024, "%"PRId64",%"PRId64, ctx->size, ctx->limit);
+        snprintf (dir_limit, 1024, "%"PRId64",%"PRId64, ctx->size, ctx->hard_lim);
 
         dict = dict_new ();
         if (dict == NULL) {
@@ -2900,10 +2935,8 @@ quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   dict_t *xdata)
 {
 	inode_t           *root_inode = NULL;
-        quota_priv_t      *priv       = NULL;
         uint64_t           value      = 0;
         quota_inode_ctx_t *ctx        = NULL;
-        limits_t          *limit_node = NULL;
 	int64_t            usage      = -1;
 	int64_t            avail      = -1;
         int64_t            blocks     = 0;
@@ -2937,33 +2970,26 @@ quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
 	usage = (ctx->size) / buf->f_bsize;
-        priv = this->private;
 
-        list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
-		/* Notice that this only works for volume-level quota. */
-                if (strcmp (limit_node->path, "/") == 0) {
-                        blocks = limit_node->value / buf->f_bsize;
-                        if (usage > blocks) {
-                                break;
-                        }
+        /* Notice that this only works for volume-level quota. */
+        blocks = ctx->hard_lim / buf->f_bsize;
+        if (usage > blocks)
+                goto unwind;
 
-                        buf->f_blocks = blocks;
-                        avail = buf->f_blocks - usage;
-                        if (buf->f_bfree > avail) {
-                                buf->f_bfree = avail;
-                        }
-                        /*
-                         * We have to assume that the total assigned quota
-                         * won't cause us to dip into the reserved space,
-                         * because dealing with the overcommitted cases is
-                         * just too hairy (especially when different bricks
-                         * might be using different reserved percentages and
-                         * such).
-                         */
-                        buf->f_bavail = buf->f_bfree;
-                        break;
-                }
-        }
+        buf->f_blocks = blocks;
+        avail = buf->f_blocks - usage;
+        if (buf->f_bfree > avail) {
+                buf->f_bfree = avail;
+        }
+        /*
+         * We have to assume that the total assigned quota
+         * won't cause us to dip into the reserved space,
+         * because dealing with the overcommitted cases is
+         * just too hairy (especially when different bricks
+         * might be using different reserved percentages and
+         * such).
+         */
+        buf->f_bavail = buf->f_bfree;
 
 unwind:
 	if (root_inode) {
@@ -3288,82 +3314,6 @@ quota_forget (xlator_t *this, inode_t *inode)
 }
 
 
-int
-quota_parse_limits (quota_priv_t *priv, xlator_t *this, dict_t *xl_options,
-                    struct list_head *old_list)
-{
-        int32_t       ret       = -1;
-        char         *str       = NULL;
-        char         *str_val   = NULL;
-        char         *path      = NULL, *saveptr = NULL;
-        uint64_t      value     = 0;
-        limits_t     *quota_lim = NULL, *old = NULL;
-        char         *last_colon= NULL;
-
-        ret = dict_get_str (xl_options, "limit-set", &str);
-
-        if (str) {
-                path = strtok_r (str, ",", &saveptr);
-
-                while (path) {
-                        last_colon = strrchr (path, ':');
-                        *last_colon = '\0';
-                        str_val = last_colon + 1;
-
-                        ret = gf_string2bytesize (str_val, &value);
-                        if (ret != 0)
-                                goto err;
-
-                        QUOTA_ALLOC_OR_GOTO (quota_lim, limits_t, err);
-
-                        quota_lim->path = path;
-
-                        quota_lim->value = value;
-
-                        gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64,
-                                quota_lim->path, quota_lim->value);
-
-                        if (old_list != NULL) {
-                                list_for_each_entry (old, old_list,
-                                                     limit_list) {
-                                        if (strcmp (old->path, quota_lim->path)
-                                            == 0) {
-                                                uuid_copy (quota_lim->gfid,
-                                                           old->gfid);
-                                                break;
-                                        }
-                                }
-                        }
-
-                        LOCK (&priv->lock);
-                        {
-                                list_add_tail (&quota_lim->limit_list,
-                                               &priv->limit_head);
-                        }
-                        UNLOCK (&priv->lock);
-
-                        path = strtok_r (NULL, ",", &saveptr);
-                }
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no \"limit-set\" option provided");
-        }
-
-        LOCK (&priv->lock);
-        {
-                list_for_each_entry (quota_lim, &priv->limit_head, limit_list) {
-                        gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64,
-                                quota_lim->path, quota_lim->value);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        ret = 0;
-err:
-        return ret;
-}
-
-
 int32_t
 init (xlator_t *this)
 {
@@ -3385,18 +3335,13 @@ init (xlator_t *this)
 
         QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err);
 
-        INIT_LIST_HEAD (&priv->limit_head);
-
-        LOCK_INIT (&priv->lock);
         this->private = priv;
 
-        ret = quota_parse_limits (priv, this, this->options, NULL);
-
-        if (ret) {
-                goto err;
-        }
-
-        GF_OPTION_INIT ("timeout", priv->timeout, int64, err);
+        GF_OPTION_INIT ("hard-timeout", priv->hard_timeout, time, err);
+        GF_OPTION_INIT ("soft-timeout", priv->soft_timeout, time, err);
+        GF_OPTION_INIT ("alert-time", priv->log_timeout, time, err);
+        GF_OPTION_INIT ("default-soft-limit", priv->default_soft_lim, percent,
+                        err);
         GF_OPTION_INIT ("deem-statfs", priv->consider_statfs, bool, err);
         GF_OPTION_INIT ("server-quota", priv->quota_on, bool, err);
 
@@ -3414,120 +3359,23 @@ err:
 }
 
 
-void
-__quota_reconfigure_inode_ctx (xlator_t *this, inode_t *inode, limits_t *limit)
-{
-        int                ret = -1;
-        quota_inode_ctx_t *ctx = NULL;
-
-        GF_VALIDATE_OR_GOTO ("quota", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, limit, out);
-
-        ret = quota_inode_ctx_get (inode, limit->value, this, NULL, NULL, &ctx,
-                                   1);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
-                        "context in inode(gfid:%s)",
-                        uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->limit = limit->value;
-        }
-        UNLOCK (&ctx->lock);
-
-out:
-        return;
-}
-
-
-void
-__quota_reconfigure (xlator_t *this, inode_table_t *itable, limits_t *limit)
-{
-        inode_t *inode = NULL;
-
-        if ((this == NULL) || (itable == NULL) || (limit == NULL)) {
-                goto out;
-        }
-
-        if (!uuid_is_null (limit->gfid)) {
-                inode = inode_find (itable, limit->gfid);
-        } else {
-                inode = inode_resolve (itable, limit->path);
-        }
-
-        if (inode != NULL) {
-                __quota_reconfigure_inode_ctx (this, inode, limit);
-        }
-
-out:
-        return;
-}
-
 
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
         int32_t           ret   = -1;
         quota_priv_t     *priv  = NULL;
-        limits_t         *limit = NULL, *next = NULL, *new = NULL;
-        struct list_head  head  = {0, };
-        xlator_t         *top   = NULL;
-        char              found = 0;
 
         priv = this->private;
         GF_VALIDATE_OR_GOTO (this->name, priv, out);
 
-        INIT_LIST_HEAD (&head);
-
-        LOCK (&priv->lock);
-        {
-                list_splice_init (&priv->limit_head, &head);
-        }
-        UNLOCK (&priv->lock);
-
-        ret = quota_parse_limits (priv, this, options, &head);
-        if (ret == -1) {
-                gf_log ("quota", GF_LOG_WARNING,
-                        "quota reconfigure failed, "
-                        "new changes will not take effect");
-                goto out;
-        }
-
-        LOCK (&priv->lock);
-        {
-                top = ((glusterfs_ctx_t *)this->ctx)->active->top;
-                GF_ASSERT (top);
-
-                list_for_each_entry (limit, &priv->limit_head, limit_list) {
-                        __quota_reconfigure (this, top->itable, limit);
-                }
-
-                list_for_each_entry_safe (limit, next, &head, limit_list) {
-                        found = 0;
-                        list_for_each_entry (new, &priv->limit_head,
-                                             limit_list) {
-                                if (strcmp (new->path, limit->path) == 0) {
-                                        found = 1;
-                                        break;
-                                }
-                        }
-
-                        if (!found) {
-                                limit->value = -1;
-                                __quota_reconfigure (this, top->itable, limit);
-                        }
-
-                        list_del_init (&limit->limit_list);
-                        GF_FREE (limit);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        GF_OPTION_RECONF ("timeout", priv->timeout, options, int64, out);
+        GF_OPTION_RECONF ("hard-timeout", priv->hard_timeout, options, time,
+                          out);
+        GF_OPTION_RECONF ("soft-timeout", priv->soft_timeout, options, time,
+                          out);
+        GF_OPTION_RECONF ("alert-time", priv->log_timeout, options, time, out);
+        GF_OPTION_RECONF ("default-soft-limit", priv->default_soft_lim,
+                          options, percent, out);
         GF_OPTION_RECONF ("deem-statfs", priv->consider_statfs, options, bool,
                           out);
         GF_OPTION_RECONF ("server-quota", priv->quota_on, options, bool, out);
@@ -3579,14 +3427,35 @@ struct xlator_cbks cbks = {
 };
 
 struct volume_options options[] = {
-        {.key = {"limit-set"}},
-        {.key = {"timeout"},
-         .type = GF_OPTION_TYPE_SIZET,
+        {.key = {"hard-timeout"},
+         .type = GF_OPTION_TYPE_TIME,
          .min = 0,
-         .max = 60,
+         .max = LONG_MAX,
          .default_value = "0",
-         .description = "quota caches the directory sizes on client. Timeout "
-                        "indicates the timeout for the cache to be revalidated."
+         .description = ""
+        },
+        {.key = {"soft-timeout"},
+         .type = GF_OPTION_TYPE_TIME,
+         .min = 0,
+         .max = LONG_MAX,
+         .default_value = "10",
+         .description = ""
+        },
+        {.key = {"alert-time"},
+         .type = GF_OPTION_TYPE_TIME,
+         .min = 0,
+         .max = LONG_MAX,
+         .default_value = "604800",
+         .description = ""
+        },
+        {.key = {"default-soft-limit"},
+         .type = GF_OPTION_TYPE_PERCENT,
+         .min = 0,
+         .max = 100,
+         .default_value = "90%",
+         .description = "Takes this if individual paths are not configured "
+                        "with soft limits. Individual limits overrides this "
+                        "option."
         },
         {.key = {"deem-statfs"},
          .type = GF_OPTION_TYPE_BOOL,