From d7b3ab5b5d17427bef5012ab72086e2b44ba1364 Mon Sep 17 00:00:00 2001 From: Raghavendra G Date: Fri, 2 Aug 2013 11:08:15 +0530 Subject: features/quota: design changes * hard and soft limits are persisted in xattrs of the inode. Associating limits with inode instead of maintaining as a global list helps us to scale better. * quotad-aggregator acts as a special client to provide cluster view through an rpc program. Quota enforcer loaded on brick uses this to get aggregated directory sizes. Aggregated sizes are cached for a timeout period in in-memory inode contexts. Signed-off-by: Raghavendra G Change-Id: I2ab508d9d4fe224bc1d8cf01cf2b7969dd4200bb BUG: 969461 --- xlators/features/quota/src/Makefile.am | 9 +- xlators/features/quota/src/quota-enforcer-client.c | 356 ++++++ xlators/features/quota/src/quota-mem-types.h | 1 + xlators/features/quota/src/quota.c | 1266 ++++++++++++-------- xlators/features/quota/src/quota.h | 118 +- xlators/features/quota/src/quotad-aggregator.c | 281 +++++ xlators/features/quota/src/quotad-aggregator.h | 37 + xlators/features/quota/src/quotad-helpers.c | 107 ++ xlators/features/quota/src/quotad-helpers.h | 24 + xlators/features/quota/src/quotad.c | 932 ++------------ 10 files changed, 1744 insertions(+), 1387 deletions(-) create mode 100644 xlators/features/quota/src/quota-enforcer-client.c create mode 100644 xlators/features/quota/src/quotad-aggregator.c create mode 100644 xlators/features/quota/src/quotad-aggregator.h create mode 100644 xlators/features/quota/src/quotad-helpers.c create mode 100644 xlators/features/quota/src/quotad-helpers.h (limited to 'xlators/features') diff --git a/xlators/features/quota/src/Makefile.am b/xlators/features/quota/src/Makefile.am index 605c198e..7165adc5 100644 --- a/xlators/features/quota/src/Makefile.am +++ b/xlators/features/quota/src/Makefile.am @@ -4,16 +4,17 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features quota_la_LDFLAGS = -module -avoid-version quotad_la_LDFLAGS = -module -avoid-version -quota_la_SOURCES = quota.c +quota_la_SOURCES = quota.c quota-enforcer-client.c quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la -quotad_la_SOURCES = quotad.c +quotad_la_SOURCES = quotad.c quotad-helpers.c quotad-aggregator.c quotad_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la -noinst_HEADERS = quota-mem-types.h quota.h +noinst_HEADERS = quota-mem-types.h quota.h quotad-aggregator.h quotad-helpers.h AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ - -I$(top_srcdir)/xlators/cluster/dht/src + -I$(top_srcdir)/xlators/cluster/dht/src -I$(top_srcdir)/rpc/xdr/src/ \ + -I$(top_srcdir)/rpc/rpc-lib/src AM_CFLAGS = -Wall $(GF_CFLAGS) diff --git a/xlators/features/quota/src/quota-enforcer-client.c b/xlators/features/quota/src/quota-enforcer-client.c new file mode 100644 index 00000000..15dc9e4d --- /dev/null +++ b/xlators/features/quota/src/quota-enforcer-client.c @@ -0,0 +1,356 @@ +/* + Copyright (c) 2010-2012 Red Hat, Inc. + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_MALLOC_H +#include +#endif + +#ifdef HAVE_MALLOC_STATS +#ifdef DEBUG +#include +#endif +#endif + +#include "quota.h" + +extern struct rpc_clnt_program quota_enforcer_clnt; + +int32_t +quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, struct iatt *postparent); + +int +quota_enforcer_submit_request (void *req, call_frame_t *frame, + rpc_clnt_prog_t *prog, + int procnum, struct iobref *iobref, + xlator_t *this, fop_cbk_fn_t cbkfn, + xdrproc_t xdrproc) +{ + int ret = -1; + int count = 0; + struct iovec iov = {0, }; + struct iobuf *iobuf = NULL; + char new_iobref = 0; + ssize_t xdr_size = 0; + quota_priv_t *priv = NULL; + + GF_ASSERT (this); + + priv = this->private; + + if (req) { + xdr_size = xdr_sizeof (xdrproc, req); + iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size); + if (!iobuf) { + goto out; + }; + + if (!iobref) { + iobref = iobref_new (); + if (!iobref) { + goto out; + } + + new_iobref = 1; + } + + iobref_add (iobref, iobuf); + + iov.iov_base = iobuf->ptr; + iov.iov_len = iobuf_size (iobuf); + + /* Create the xdr payload */ + ret = xdr_serialize_generic (iov, req, xdrproc); + if (ret == -1) { + goto out; + } + iov.iov_len = ret; + count = 1; + } + + /* Send the msg */ + ret = rpc_clnt_submit (priv->rpc_clnt, prog, procnum, cbkfn, + &iov, count, + NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL); + ret = 0; + +out: + if (new_iobref) + iobref_unref (iobref); + if (iobuf) + iobuf_unref (iobuf); + + return ret; +} + +int +quota_enforcer_lookup_cbk (struct rpc_req *req, struct iovec *iov, + int count, void *myframe) +{ + quota_local_t *local = NULL; + call_frame_t *frame = NULL; + int ret = 0; + gfs3_lookup_rsp rsp = {0,}; + struct iatt stbuf = {0,}; + struct iatt postparent = {0,}; + int op_errno = EINVAL; + dict_t *xdata = NULL; + inode_t *inode = NULL; + xlator_t *this = NULL; + + this = THIS; + + frame = myframe; + local = frame->local; + inode = local->validate_loc.inode; + + if (-1 == req->rpc_status) { + rsp.op_ret = -1; + op_errno = ENOTCONN; + goto out; + } + + ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lookup_rsp); + if (ret < 0) { + gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed"); + rsp.op_ret = -1; + op_errno = EINVAL; + goto out; + } + + op_errno = gf_error_to_errno (rsp.op_errno); + gf_stat_to_iatt (&rsp.postparent, &postparent); + + if (rsp.op_ret == -1) + goto out; + + rsp.op_ret = -1; + gf_stat_to_iatt (&rsp.stat, &stbuf); + + GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata, (rsp.xdata.xdata_val), + (rsp.xdata.xdata_len), rsp.op_ret, + op_errno, out); + + if ((!uuid_is_null (inode->gfid)) + && (uuid_compare (stbuf.ia_gfid, inode->gfid) != 0)) { + gf_log (frame->this->name, GF_LOG_DEBUG, + "gfid changed for %s", local->validate_loc.path); + rsp.op_ret = -1; + op_errno = ESTALE; + goto out; + } + + rsp.op_ret = 0; + +out: + rsp.op_errno = op_errno; + if (rsp.op_ret == -1) { + /* any error other than ENOENT */ + if (rsp.op_errno != ENOENT) + gf_log (this->name, GF_LOG_WARNING, + "remote operation failed: %s. Path: %s (%s)", + strerror (rsp.op_errno), + local->validate_loc.path, + loc_gfid_utoa (&local->validate_loc)); + else + gf_log (this->name, GF_LOG_TRACE, + "not found on remote node"); + + } + + local->validate_cbk (frame, NULL, this, rsp.op_ret, rsp.op_errno, inode, + &stbuf, xdata, &postparent); + + if (xdata) + dict_unref (xdata); + + free (rsp.xdata.xdata_val); + + return 0; +} + +int +quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, + dict_t *xdata, fop_lookup_cbk_t validate_cbk) +{ + quota_local_t *local = NULL; + gfs3_lookup_req req = {{0,},}; + int ret = 0; + int op_errno = ESTALE; + struct iovec vector[MAX_IOVEC] = {{0}, }; + quota_priv_t *priv = NULL; + + if (!frame || !this || !loc) + goto unwind; + + local = frame->local; + local->validate_cbk = validate_cbk; + + memset (vector, 0, sizeof (vector)); + + priv = this->private; + + if (!(loc && loc->inode)) + goto unwind; + + if (loc->parent) { + if (!uuid_is_null (loc->parent->gfid)) + memcpy (req.pargfid, loc->parent->gfid, 16); + else + memcpy (req.pargfid, loc->pargfid, 16); + } else { + if (!uuid_is_null (loc->inode->gfid)) + memcpy (req.gfid, loc->inode->gfid, 16); + else + memcpy (req.gfid, loc->gfid, 16); + } + + if (xdata) { + GF_PROTOCOL_DICT_SERIALIZE (this, xdata, + (&req.xdata.xdata_val), + req.xdata.xdata_len, + op_errno, unwind); + } + + if (loc->name) + req.bname = (char *)loc->name; + else + req.bname = ""; + + ret = quota_enforcer_submit_request (&req, frame, + priv->quota_enforcer, + GF_AGGREGATOR_LOOKUP, + NULL, this, + quota_enforcer_lookup_cbk, + (xdrproc_t)xdr_gfs3_lookup_req); + + if (ret) { + gf_log (this->name, GF_LOG_WARNING, "failed to send the fop"); + } + + GF_FREE (req.xdata.xdata_val); + + return 0; + +unwind: + validate_cbk (frame, NULL, this, -1, op_errno, NULL, NULL, NULL, NULL); + + GF_FREE (req.xdata.xdata_val); + + return 0; +} + +int +quota_enforcer_notify (struct rpc_clnt *rpc, void *mydata, + rpc_clnt_event_t event, void *data) +{ + xlator_t *this = NULL; + int ret = 0; + + this = mydata; + + switch (event) { + case RPC_CLNT_CONNECT: + { + gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT"); + break; + } + + case RPC_CLNT_DISCONNECT: + { + gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT"); + break; + } + + default: + gf_log (this->name, GF_LOG_TRACE, + "got some other RPC event %d", event); + ret = 0; + break; + } + + return ret; +} + +struct rpc_clnt * +quota_enforcer_init (xlator_t *this, dict_t *options) +{ + struct rpc_clnt *rpc = NULL; + quota_priv_t *priv = NULL; + int ret = -1; + + priv = this->private; + priv->quota_enforcer = "a_enforcer_clnt; + + dict_set_str (options, "transport.address-family", "unix"); + dict_set_str (options, "transport-type", "socket"); + dict_set_str (options, "transport.socket.connect-path", + "/tmp/quotad.socket"); + + rpc = rpc_clnt_new (options, this->ctx, this->name, 16); + if (!rpc) + goto out; + + ret = rpc_clnt_register_notify (rpc, quota_enforcer_notify, this); + if (ret) { + gf_log ("cli", GF_LOG_ERROR, "failed to register notify"); + goto out; + } + + rpc_clnt_start (rpc); +out: + if (ret) { + if (rpc) + rpc_clnt_unref (rpc); + rpc = NULL; + } + + priv->rpc_clnt = rpc; + return rpc; +} + +struct rpc_clnt_procedure quota_enforcer_actors[GF_AGGREGATOR_MAXVALUE] = { + [GF_AGGREGATOR_NULL] = {"NULL", NULL}, + [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", NULL}, +}; + +struct rpc_clnt_program quota_enforcer_clnt = { + .progname = "Quota enforcer", + .prognum = GLUSTER_AGGREGATOR_PROGRAM, + .progver = GLUSTER_AGGREGATOR_VERSION, + .numproc = GF_AGGREGATOR_MAXVALUE, + .proctable = quota_enforcer_actors, +}; diff --git a/xlators/features/quota/src/quota-mem-types.h b/xlators/features/quota/src/quota-mem-types.h index 4831476d..97d91656 100644 --- a/xlators/features/quota/src/quota-mem-types.h +++ b/xlators/features/quota/src/quota-mem-types.h @@ -23,6 +23,7 @@ enum gf_quota_mem_types_ { gf_quota_mt_quota_dentry_t, gf_quota_mt_quota_limits_level_t, gf_quota_mt_qd_vols_conf_t, + gf_quota_mt_aggregator_state_t, gf_quota_mt_end }; #endif diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c index 744748fd..bf0666cb 100644 --- a/xlators/features/quota/src/quota.c +++ b/xlators/features/quota/src/quota.c @@ -230,32 +230,183 @@ out: return; } +inline void +quota_resume_fop_if_validation_done (quota_local_t *local) +{ + call_stub_t *stub = NULL; + int link_count = -1; + + if (local == NULL) + goto out; + + LOCK (&local->lock); + { + link_count = local->link_count; + if (link_count == 0) { + stub = local->stub; + local->stub = NULL; + } + } + UNLOCK (&local->lock); + + if (stub != NULL) { + call_resume (stub); + } +out: + return; +} + +inline void +quota_handle_validate_error (quota_local_t *local, int32_t op_ret, + int32_t op_errno) +{ + if (local == NULL) + goto out; + + LOCK (&local->lock); + { + if (op_ret < 0) { + local->op_ret = op_ret; + local->op_errno = op_errno; + } + + /* we abort checking limits on this path to root */ + local->link_count--; + } + UNLOCK (&local->lock); + + quota_resume_fop_if_validation_done (local); +out: + return; +} + +int32_t +quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, struct iatt *postparent) +{ + quota_local_t *local = NULL; + int32_t ret = 0; + quota_inode_ctx_t *ctx = NULL; + int64_t *size = 0; + uint64_t value = 0; + + local = frame->local; + + if (op_ret < 0) { + goto unwind; + } + + GF_ASSERT (local); + GF_ASSERT (frame); + GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, unwind, op_errno, + EINVAL); + GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, unwind, op_errno, + EINVAL); + + ret = inode_ctx_get (local->validate_loc.inode, this, &value); + + ctx = (quota_inode_ctx_t *)(unsigned long)value; + if ((ret == -1) || (ctx == NULL)) { + gf_log (this->name, GF_LOG_WARNING, + "quota context is not present in inode (gfid:%s)", + uuid_utoa (local->validate_loc.inode->gfid)); + op_errno = EINVAL; + goto unwind; + } + + ret = dict_get_bin (xdata, QUOTA_SIZE_KEY, (void **) &size); + if (ret < 0) { + gf_log (this->name, GF_LOG_WARNING, + "size key not present in dict"); + op_errno = EINVAL; + goto unwind; + } + + local->just_validated = 1; /* so that we don't go into infinite + * loop of validation and checking + * limit when timeout is zero. + */ + LOCK (&ctx->lock); + { + ctx->size = ntoh64 (*size); + gettimeofday (&ctx->tv, NULL); + } + UNLOCK (&ctx->lock); + + quota_check_limit (frame, local->validate_loc.inode, this, NULL, NULL); + return 0; + +unwind: + quota_handle_validate_error (local, op_ret, op_errno); + return 0; +} + + +static inline uint64_t +quota_time_elapsed (struct timeval *now, struct timeval *then) +{ + return (now->tv_sec - then->tv_sec); +} + + +int32_t +quota_timeout (struct timeval *tv, int32_t timeout) +{ + struct timeval now = {0,}; + int32_t timed_out = 0; + + gettimeofday (&now, NULL); + + if (quota_time_elapsed (&now, tv) >= timeout) { + timed_out = 1; + } + + return timed_out; +} + int32_t quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this, char *name, uuid_t par) { - inode_t *_inode = NULL, *parent = NULL; - quota_inode_ctx_t *ctx = NULL; - quota_local_t *local = NULL; - char need_unwind = 0; - int64_t delta = 0; - uint64_t value = 0; - uuid_t trav_uuid = {0,}; - - GF_VALIDATE_OR_GOTO ("quota", this, out); - GF_VALIDATE_OR_GOTO (this->name, frame, out); - GF_VALIDATE_OR_GOTO (this->name, inode, out); + int32_t ret = -1, op_errno = EINVAL; + inode_t *_inode = NULL, *parent = NULL; + quota_inode_ctx_t *ctx = NULL; + quota_priv_t *priv = NULL; + quota_local_t *local = NULL; + char need_validate = 0; + gf_boolean_t soft_limit_exceeded = 0, hard_limit_exceeded = 0; + int64_t delta = 0, wouldbe_size = 0; + uint64_t value = 0; + char just_validated = 0; + uuid_t trav_uuid = {0,}; + dict_t *xdata = NULL; + uint32_t timeout = 0; + + GF_VALIDATE_OR_GOTO ("quota", this, err); + GF_VALIDATE_OR_GOTO (this->name, frame, err); + GF_VALIDATE_OR_GOTO (this->name, inode, err); local = frame->local; - GF_VALIDATE_OR_GOTO (this->name, local, out); + GF_VALIDATE_OR_GOTO (this->name, local, err); delta = local->delta; + GF_VALIDATE_OR_GOTO (this->name, local->stub, err); + + priv = this->private; + inode_ctx_get (inode, this, &value); ctx = (quota_inode_ctx_t *)(unsigned long)value; _inode = inode_ref (inode); + LOCK (&local->lock); + { + just_validated = local->just_validated; + local->just_validated = 0; + } + UNLOCK (&local->lock); if ( par != NULL ) { uuid_copy (trav_uuid, par); @@ -263,25 +414,53 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this, do { if (ctx != NULL) { + wouldbe_size = ctx->size + delta; + LOCK (&ctx->lock); { - if (ctx->hard_lim >= 0) { - if ((ctx->size + delta) - >= ctx->hard_lim) { - local->op_ret = -1; - local->op_errno = EDQUOT; - need_unwind = 1; - } + timeout = priv->soft_timeout; + + if ((ctx->soft_lim >= 0) + && (wouldbe_size > ctx->soft_lim)) { + timeout = priv->hard_timeout; + } + + if (!just_validated + && quota_timeout (&ctx->tv, timeout)) { + need_validate = 1; + } else if (wouldbe_size >= ctx->hard_lim) { + hard_limit_exceeded = 1; + } else if (wouldbe_size >= ctx->soft_lim) { + soft_limit_exceeded = 1; } } UNLOCK (&ctx->lock); - if (need_unwind) { - break; + if (need_validate) { + goto validate; + } + + if (hard_limit_exceeded) { + op_errno = EDQUOT; + goto err; + } + + if (soft_limit_exceeded) { + gf_log (this->name, GF_LOG_WARNING, + "size (%"PRId64") of directory (%s) " + "has exceeded soft-limit (%"PRId64")", + wouldbe_size, uuid_utoa (_inode->gfid), + ctx->soft_lim); } } if (__is_root_gfid (_inode->gfid)) { + LOCK (&local->lock); + { + --local->link_count; + } + UNLOCK (&local->lock); + break; } @@ -301,6 +480,7 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this, inode_unref (_inode); _inode = parent; + just_validated = 0; if (_inode == NULL) { break; @@ -311,49 +491,71 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this, ctx = (quota_inode_ctx_t *)(unsigned long)value; } while (1); - inode_unref (_inode); + if (_inode != NULL) { + inode_unref (_inode); + } -out: - return local->op_ret; -} + quota_resume_fop_if_validation_done (local); + return 0; +validate: + LOCK (&local->lock); + { + loc_wipe (&local->validate_loc); -int32_t -quota_get_limit_value (inode_t *inode, xlator_t *this, int64_t *n) -{ - int32_t ret = 0; - char *path = NULL; - limits_t *limit_node = NULL; - quota_priv_t *priv = NULL; + ret = quota_inode_loc_fill (_inode, &local->validate_loc); + if (ret < 0) { + gf_log (this->name, GF_LOG_WARNING, + "cannot fill loc for inode (gfid:%s), hence " + "aborting quota-checks and continuing with fop", + uuid_utoa (_inode->gfid)); + } + } + UNLOCK (&local->lock); - if (inode == NULL || n == NULL) { - ret = -1; - goto out; + if (ret < 0) { + op_errno = ENOMEM; + goto err; } - *n = 0; + xdata = dict_new (); + if (xdata == NULL) { + op_errno = ENOMEM; + goto err; + } - ret = inode_path (inode, NULL, &path); + ret = dict_set_int8 (xdata, QUOTA_SIZE_KEY, 1); if (ret < 0) { - ret = -1; - goto out; + gf_log (this->name, GF_LOG_WARNING, "dict set failed"); + op_errno = ENOMEM; + goto err; } - priv = this->private; + ret = dict_set_str (xdata, "volume-uuid", priv->volume_uuid); + if (ret < 0) { + gf_log (this->name, GF_LOG_WARNING, "dict set failed"); + op_errno = ENOMEM; + goto err; + } - list_for_each_entry (limit_node, &priv->limit_head, limit_list) { - if (strcmp (limit_node->path, path) == 0) { - *n = limit_node->hard_lim; - break; - } + ret = quota_enforcer_lookup (frame, this, &local->validate_loc, xdata, + quota_validate_cbk); + + if (ret < 0) { + op_errno = ENOTCONN; + goto err; } -out: - GF_FREE (path); + inode_unref (_inode); - return ret; -} + return 0; +err: + quota_handle_validate_error (local, -1, op_errno); + + inode_unref (_inode); + return 0; +} static int32_t quota_inode_ctx_get (inode_t *inode, int64_t hard_lim, int64_t soft_lim, @@ -387,46 +589,46 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, { int32_t ret = -1; char found = 0; - quota_local_t *local = NULL; quota_inode_ctx_t *ctx = NULL; quota_dentry_t *dentry = NULL; uint64_t value = 0; - limits_t *limit_node = NULL; - quota_priv_t *priv = NULL; int64_t *size = NULL; + int64_t hard_lim = -1, soft_lim = -1, *ptr = NULL; + quota_local_t *local = NULL; - local = frame->local; + if (op_ret < 0) + goto unwind; - priv = this->private; + if (dict != NULL) { + ret = dict_get_bin (dict, QUOTA_SOFT_LIMIT_KEY, (void **) &ptr); + if (ptr != NULL) { + soft_lim = ntoh64 (*ptr); + } + + ptr = NULL; + ret = dict_get_bin (dict, QUOTA_HARD_LIMIT_KEY, (void **) &ptr); + if (ptr != NULL) { + hard_lim = ntoh64 (*ptr); + } + } + + local = frame->local; inode_ctx_get (inode, this, &value); ctx = (quota_inode_ctx_t *)(unsigned long)value; - if ((op_ret < 0) || (local == NULL) - || (((ctx == NULL) || (ctx->hard_lim == local->hard_lim)) - && (local->hard_lim < 0) && !((IA_ISREG (buf->ia_type)) + if ((((ctx == NULL) || (ctx->hard_lim == hard_lim)) + && (hard_lim < 0) && !((IA_ISREG (buf->ia_type)) || (IA_ISLNK (buf->ia_type))))) { goto unwind; } - LOCK (&priv->lock); - { - list_for_each_entry (limit_node, &priv->limit_head, - limit_list) { - if (strcmp (local->loc.path, limit_node->path) == 0) { - uuid_copy (limit_node->gfid, buf->ia_gfid); - break; - } - } - } - UNLOCK (&priv->lock); - - ret = quota_inode_ctx_get (local->loc.inode, local->hard_lim, - local->soft_lim, this, dict, buf, &ctx, 1); + ret = quota_inode_ctx_get (inode, hard_lim, soft_lim, + this, dict, buf, &ctx, 1); if ((ret == -1) || (ctx == NULL)) { gf_log (this->name, GF_LOG_WARNING, "cannot create quota " "context in inode(gfid:%s)", - uuid_utoa (local->loc.inode->gfid)); + uuid_utoa (inode->gfid)); op_ret = -1; op_errno = ENOMEM; goto unwind; @@ -434,8 +636,8 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, LOCK (&ctx->lock); { - ctx->hard_lim = local->hard_lim; - ctx->soft_lim = local->soft_lim; + ctx->hard_lim = hard_lim; + ctx->soft_lim = soft_lim; ctx->buf = *buf; @@ -494,43 +696,42 @@ int32_t quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req) { - quota_priv_t *priv = NULL; - int32_t ret = -1; - limits_t *limit_node = NULL; - quota_local_t *local = NULL; - int64_t hard_lim = -1; - int64_t soft_lim = -1; + quota_priv_t *priv = NULL; + int32_t ret = -1; + quota_local_t *local = NULL; priv = this->private; WIND_IF_QUOTAOFF (priv->is_quota_on, wind); - - list_for_each_entry (limit_node, &priv->limit_head, limit_list) { - if (strcmp (limit_node->path, loc->path) == 0) { - hard_lim = limit_node->hard_lim; - soft_lim = limit_node->soft_lim; - break; + if (priv->is_quota_on) { + local = quota_local_new (); + if (local == NULL) { + goto err; } - } - - local = quota_local_new (); - if (local == NULL) { - goto err; - } - ret = loc_copy (&local->loc, loc); - if (ret == -1) { - goto err; - } + frame->local = local; + loc_copy (&local->loc, loc); - frame->local = local; + if (xattr_req == NULL) { + xattr_req = dict_new (); + if (xattr_req == NULL) + goto err; + } - local->hard_lim = hard_lim; - local->soft_lim = soft_lim; + ret = dict_set_int8 (xattr_req, QUOTA_HARD_LIMIT_KEY, 1); + if (ret < 0) { + gf_log (this->name, GF_LOG_WARNING, + "dict set of key for hard-limit failed"); + goto err; + } - if (hard_lim < 0) { - goto wind; + ret = dict_set_int8 (xattr_req, QUOTA_SOFT_LIMIT_KEY, 1); + if (ret < 0) { + gf_log (this->name, GF_LOG_WARNING, + "dict set of key for soft-limit failed"); + goto err; + } } wind: @@ -673,6 +874,40 @@ out: } +int32_t +quota_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, + struct iovec *vector, int32_t count, off_t off, + uint32_t flags, struct iobref *iobref, dict_t *xdata) +{ + quota_local_t *local = NULL; + int32_t op_errno = EINVAL; + quota_priv_t *priv = NULL; + + priv = this->private; + + local = frame->local; + if (local == NULL) { + gf_log (this->name, GF_LOG_WARNING, "local is NULL"); + goto unwind; + } + + if (local->op_ret == -1) { + op_errno = local->op_errno; + goto unwind; + } + + STACK_WIND (frame, + priv->is_quota_on? quota_writev_cbk: default_writev_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev, fd, + vector, count, off, flags, iobref, xdata); + return 0; + +unwind: + QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL); + return 0; +} + + int32_t quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector, int32_t count, off_t off, @@ -685,6 +920,7 @@ quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, quota_local_t *local = NULL; quota_inode_ctx_t *ctx = NULL; quota_dentry_t *dentry = NULL; + call_stub_t *stub = NULL; priv = this->private; @@ -711,6 +947,13 @@ quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, goto unwind; } + stub = fop_writev_stub (frame, quota_writev_helper, fd, vector, count, + off, flags, iobref, xdata); + if (stub == NULL) { + op_errno = ENOMEM; + goto unwind; + } + priv = this->private; GF_VALIDATE_OR_GOTO (this->name, priv, unwind); @@ -725,26 +968,24 @@ quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, local->delta = size; local->link_count = parents; + local->stub = stub; list_for_each_entry (dentry, &ctx->parents, next) { - ret = quota_check_limit (frame, fd->inode, this, dentry->name, - dentry->par); - if (ret == -1) { - op_errno = EDQUOT; - goto unwind; - } + quota_check_limit (frame, fd->inode, this, dentry->name, + dentry->par); } + return 0; + +unwind: + QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL); + return 0; + wind: STACK_WIND (frame, priv->is_quota_on? quota_writev_cbk: default_writev_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags, iobref, xdata); - - return 0; - -unwind: - QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL); return 0; } @@ -761,13 +1002,49 @@ quota_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, } +int32_t +quota_mkdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, + mode_t mode, mode_t umask, dict_t *xdata) +{ + quota_local_t *local = NULL; + int32_t op_errno = EINVAL; + quota_priv_t *priv = NULL; + + local = frame->local; + if (local == NULL) { + gf_log (this->name, GF_LOG_WARNING, "local is NULL"); + goto unwind; + } + + priv = this->private; + op_errno = local->op_errno; + + if (local->op_ret == -1) { + goto unwind; + } + + STACK_WIND (frame, + priv->is_quota_on? quota_mkdir_cbk: default_mkdir_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc, + mode, umask, xdata); + + return 0; + +unwind: + QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, + NULL, NULL, NULL); + return 0; +} + + int32_t quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata) { - quota_priv_t *priv = NULL; - int32_t ret = 0, op_errno = 0; - quota_local_t *local = NULL; + quota_priv_t *priv = NULL; + int32_t ret = 0, op_errno = 0; + quota_local_t *local = NULL; + call_stub_t *stub = NULL; priv = this->private; @@ -781,8 +1058,6 @@ quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, frame->local = local; - local->link_count = 1; - ret = loc_copy (&local->loc, loc); if (ret) { op_errno = ENOMEM; @@ -790,26 +1065,33 @@ quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, goto err; } - local->delta = 0; - - ret = quota_check_limit (frame, loc->parent, this, NULL, NULL); - if (ret == -1) { - op_errno = EDQUOT; + stub = fop_mkdir_stub (frame, quota_mkdir_helper, loc, mode, umask, + xdata); + if (stub == NULL) { + op_errno = ENOMEM; goto err; } -wind: - STACK_WIND (frame, - priv->is_quota_on? quota_mkdir_cbk: default_mkdir_cbk, - FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc, - mode, umask, xdata); + local->stub = stub; + local->delta = 0; + local->link_count = 1; + quota_check_limit (frame, loc->parent, this, NULL, NULL); return 0; + err: QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL); return 0; + +wind: + STACK_WIND (frame, + priv->is_quota_on? quota_mkdir_cbk: default_mkdir_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc, + mode, umask, xdata); + + return 0; } @@ -865,14 +1147,52 @@ unwind: } +int32_t +quota_create_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, + int32_t flags, mode_t mode, mode_t umask, fd_t *fd, + dict_t *xdata) +{ + quota_local_t *local = NULL; + int32_t op_errno = EINVAL; + quota_priv_t *priv = NULL; + + local = frame->local; + + priv = this->private; + + if (local == NULL) { + gf_log (this->name, GF_LOG_WARNING, "local is NULL"); + goto unwind; + } + + if (local->op_ret == -1) { + op_errno = local->op_errno; + goto unwind; + } + + + STACK_WIND (frame, + priv->is_quota_on? quota_create_cbk: default_create_cbk, + FIRST_CHILD (this), FIRST_CHILD (this)->fops->create, loc, + flags, mode, umask, fd, xdata); + return 0; + +unwind: + QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, + NULL, NULL, NULL, NULL); + return 0; +} + + int32_t quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata) { - quota_priv_t *priv = NULL; - int32_t ret = -1; - quota_local_t *local = NULL; - int32_t op_errno = 0; + quota_priv_t *priv = NULL; + int32_t ret = -1; + quota_local_t *local = NULL; + int32_t op_errno = 0; + call_stub_t *stub = NULL; priv = this->private; @@ -893,26 +1213,30 @@ quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, goto err; } - local->delta = 0; - - ret = quota_check_limit (frame, loc->parent, this, NULL, NULL); - if (ret == -1) { - op_errno = EDQUOT; + stub = fop_create_stub (frame, quota_create_helper, loc, flags, mode, + umask, fd, xdata); + if (stub == NULL) { goto err; } -wind: - STACK_WIND (frame, - priv->is_quota_on? quota_create_cbk: default_create_cbk, - FIRST_CHILD (this), FIRST_CHILD (this)->fops->create, loc, - flags, mode, umask, fd, xdata); + local->link_count = 1; + local->stub = stub; + local->delta = 0; + quota_check_limit (frame, loc->parent, this, NULL, NULL); return 0; err: QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL, NULL); return 0; + +wind: + STACK_WIND (frame, + priv->is_quota_on? quota_create_cbk: default_create_cbk, + FIRST_CHILD (this), FIRST_CHILD (this)->fops->create, loc, + flags, mode, umask, fd, xdata); + return 0; } @@ -1086,14 +1410,47 @@ out: } +int32_t +quota_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc, + loc_t *newloc, dict_t *xdata) +{ + quota_local_t *local = NULL; + int32_t op_errno = EINVAL; + quota_priv_t *priv = NULL; + + local = frame->local; + if (local == NULL) { + gf_log (this->name, GF_LOG_WARNING, "local is NULL"); + goto unwind; + } + + op_errno = local->op_errno; + + if (local->op_ret == -1) { + goto unwind; + } + + STACK_WIND (frame, priv->is_quota_on? quota_link_cbk: default_link_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->link, oldloc, + newloc, xdata); + return 0; + +unwind: + QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, + NULL, NULL, NULL); + return 0; +} + + int32_t quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata) { - quota_priv_t *priv = NULL; + quota_priv_t *priv = NULL; int32_t ret = -1, op_errno = ENOMEM; quota_local_t *local = NULL; - quota_inode_ctx_t *ctx = NULL; + quota_inode_ctx_t *ctx = NULL; + call_stub_t *stub = NULL; priv = this->private; @@ -1123,21 +1480,28 @@ quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, goto err; } + stub = fop_link_stub (frame, quota_link_helper, oldloc, newloc, xdata); + if (stub == NULL) { + goto err; + } + + local->link_count = 1; + local->stub = stub; local->delta = ctx->buf.ia_blocks * 512; - ret = quota_check_limit (frame, newloc->parent, this, NULL, NULL); - if (ret == -1) { - op_errno = EDQUOT; + ret = quota_inode_ctx_get (oldloc->inode, -1, -1, this, NULL, NULL, + &ctx, 0); + if (ctx == NULL) { + gf_log (this->name, GF_LOG_WARNING, + "quota context not set in inode (gfid:%s)", + oldloc->inode ? uuid_utoa (oldloc->inode->gfid) : "0"); + op_errno = EINVAL; goto err; } -wind: - STACK_WIND (frame, - priv->is_quota_on? quota_link_cbk: default_link_cbk, - FIRST_CHILD(this), FIRST_CHILD(this)->fops->link, oldloc, - newloc, xdata); + quota_check_limit (frame, newloc->parent, this, NULL, NULL); + return 0; - ret = 0; err: if (ret < 0) { QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, @@ -1145,6 +1509,12 @@ err: } return 0; + +wind: + STACK_WIND (frame, priv->is_quota_on? quota_link_cbk: default_link_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->link, oldloc, + newloc, xdata); + return 0; } @@ -1266,14 +1636,51 @@ out: } +int32_t +quota_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc, + loc_t *newloc, dict_t *xdata) +{ + quota_local_t *local = NULL; + int32_t op_errno = EINVAL; + quota_priv_t *priv = NULL; + + priv = this->private; + + local = frame->local; + if (local == NULL) { + gf_log (this->name, GF_LOG_WARNING, "local is NULL"); + goto unwind; + } + + op_errno = local->op_errno; + + if (local->op_ret == -1) { + goto unwind; + } + + STACK_WIND (frame, + priv->is_quota_on? quota_rename_cbk: default_rename_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename, oldloc, + newloc, xdata); + + return 0; + +unwind: + QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, + NULL, NULL, NULL, NULL); + return 0; +} + + int32_t quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata) { - quota_priv_t *priv = NULL; - int32_t ret = -1, op_errno = ENOMEM; - quota_local_t *local = NULL; - quota_inode_ctx_t *ctx = NULL; + quota_priv_t *priv = NULL; + int32_t ret = -1, op_errno = ENOMEM; + quota_local_t *local = NULL; + quota_inode_ctx_t *ctx = NULL; + call_stub_t *stub = NULL; priv = this->private; @@ -1298,6 +1705,15 @@ quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, goto err; } + stub = fop_rename_stub (frame, quota_rename_helper, oldloc, newloc, + xdata); + if (stub == NULL) { + goto err; + } + + local->link_count = 1; + local->stub = stub; + if (IA_ISREG (oldloc->inode->ia_type) || IA_ISLNK (oldloc->inode->ia_type)) { ret = quota_inode_ctx_get (oldloc->inode, -1, -1, this, NULL, @@ -1315,11 +1731,13 @@ quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, local->delta = 0; } - ret = quota_check_limit (frame, newloc->parent, this, NULL, NULL); - if (ret == -1) { - op_errno = EDQUOT; - goto err; - } + quota_check_limit (frame, newloc->parent, this, NULL, NULL); + return 0; + +err: + QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL, + NULL, NULL, NULL, NULL, NULL); + return 0; wind: STACK_WIND (frame, @@ -1327,13 +1745,6 @@ wind: FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata); - ret = 0; -err: - if (ret == -1) { - QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL, - NULL, NULL, NULL, NULL, NULL); - } - return 0; } @@ -1384,10 +1795,44 @@ quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this, } UNLOCK (&ctx->lock); -out: - QUOTA_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf, - preparent, postparent, xdata); +out: + QUOTA_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf, + preparent, postparent, xdata); + + return 0; +} + + +int +quota_symlink_helper (call_frame_t *frame, xlator_t *this, const char *linkpath, + loc_t *loc, mode_t umask, dict_t *xdata) +{ + quota_local_t *local = NULL; + int32_t op_errno = EINVAL; + quota_priv_t *priv = NULL; + + local = frame->local; + if (local == NULL) { + gf_log (this->name, GF_LOG_WARNING, "local is NULL"); + goto unwind; + } + + priv = this->private; + + if (local->op_ret == -1) { + op_errno = local->op_errno; + goto unwind; + } + + STACK_WIND (frame, + priv->is_quota_on? quota_symlink_cbk: default_symlink_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink, + linkpath, loc, umask, xdata); + return 0; +unwind: + QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, + NULL, NULL, NULL); return 0; } @@ -1396,16 +1841,16 @@ int quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata) { - quota_priv_t *priv = NULL; - int32_t ret = -1; - int32_t op_errno = ENOMEM; - quota_local_t *local = NULL; + quota_priv_t *priv = NULL; + int32_t ret = -1; + int32_t op_errno = ENOMEM; + quota_local_t *local = NULL; + call_stub_t *stub = NULL; priv = this->private; WIND_IF_QUOTAOFF (priv->is_quota_on, wind); - local = quota_local_new (); if (local == NULL) { goto err; @@ -1419,19 +1864,17 @@ quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath, goto err; } - local->delta = strlen (linkpath); - - ret = quota_check_limit (frame, loc->parent, this, NULL, NULL); - if (ret == -1) { - op_errno = EDQUOT; + stub = fop_symlink_stub (frame, quota_symlink_helper, linkpath, loc, + umask, xdata); + if (stub == NULL) { goto err; } -wind: - STACK_WIND (frame, - priv->is_quota_on? quota_symlink_cbk: default_symlink_cbk, - FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink, - linkpath, loc, umask, xdata); + local->stub = stub; + local->delta = strlen (linkpath); + local->link_count = 1; + + quota_check_limit (frame, loc->parent, this, NULL, NULL); return 0; err: @@ -1439,6 +1882,13 @@ err: NULL, NULL); return 0; + +wind: + STACK_WIND (frame, + priv->is_quota_on? quota_symlink_cbk: default_symlink_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink, + linkpath, loc, umask, xdata); + return 0; } @@ -2270,14 +2720,49 @@ unwind: } +int +quota_mknod_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, + mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata) +{ + quota_local_t *local = NULL; + int32_t op_errno = EINVAL; + quota_priv_t *priv = NULL; + + local = frame->local; + if (local == NULL) { + gf_log (this->name, GF_LOG_WARNING, "local is NULL"); + goto unwind; + } + + priv = this->private; + + if (local->op_ret == -1) { + op_errno = local->op_errno; + goto unwind; + } + + STACK_WIND (frame, + priv->is_quota_on? quota_mknod_cbk: default_mknod_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod, loc, + mode, rdev, umask, xdata); + + return 0; + +unwind: + QUOTA_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, + NULL, NULL, NULL); + return 0; +} + + int quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata) { - quota_priv_t *priv = NULL; - int32_t ret = -1; - quota_local_t *local = NULL; - int32_t op_errno = 0; + quota_priv_t *priv = NULL; + int32_t ret = -1; + quota_local_t *local = NULL; + call_stub_t *stub = NULL; priv = this->private; @@ -2296,29 +2781,35 @@ quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, goto err; } - local->delta = 0; - - ret = quota_check_limit (frame, loc->parent, this, NULL, NULL); - if (ret == -1) { - op_errno = EDQUOT; + stub = fop_mknod_stub (frame, quota_mknod_helper, loc, mode, rdev, + umask, xdata); + if (stub == NULL) { goto err; } + local->link_count = 1; + local->stub = stub; + local->delta = 0; + + quota_check_limit (frame, loc->parent, this, NULL, NULL); + return 0; +err: + QUOTA_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, + NULL); + + return 0; + wind: STACK_WIND (frame, priv->is_quota_on? quota_mknod_cbk: default_mknod_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata); - return 0; -err: - QUOTA_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, - NULL); - return 0; } + int quota_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, dict_t *xdata) @@ -2509,13 +3000,6 @@ quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, dict_t *xdata) { inode_t *root_inode = NULL; - quota_priv_t *priv = NULL; - uint64_t value = 0; - quota_inode_ctx_t *ctx = NULL; - limits_t *limit_node = NULL; - int64_t usage = -1; - int64_t avail = -1; - int64_t blocks = 0; root_inode = cookie; @@ -2538,40 +3022,13 @@ quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, "non-root inode, cannot adjust for quota"); goto unwind; } - +/* inode_ctx_get (root_inode, this, &value); if (!value) { goto unwind; } ctx = (quota_inode_ctx_t *)(unsigned long)value; - usage = (ctx->size) / buf->f_bsize; - priv = this->private; - - list_for_each_entry (limit_node, &priv->limit_head, limit_list) { - /* Notice that this only works for volume-level quota. */ - if (strcmp (limit_node->path, "/") == 0) { - blocks = limit_node->hard_lim / buf->f_bsize; - if (usage > blocks) { - break; - } - - buf->f_blocks = blocks; - avail = buf->f_blocks - usage; - if (buf->f_bfree > avail) { - buf->f_bfree = avail; - } - /* - * We have to assume that the total assigned quota - * won't cause us to dip into the reserved space, - * because dealing with the overcommitted cases is - * just too hairy (especially when different bricks - * might be using different reserved percentages and - * such). - */ - buf->f_bavail = buf->f_bfree; - break; - } - } +*/ unwind: if (root_inode) { @@ -2722,6 +3179,41 @@ out: return 0; } + +int32_t +quota_fallocate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, + int32_t mode, off_t offset, size_t len, dict_t *xdata) +{ + quota_local_t *local = NULL; + int32_t op_errno = EINVAL; + quota_priv_t *priv = NULL; + + local = frame->local; + if (local == NULL) { + gf_log (this->name, GF_LOG_WARNING, "local is NULL"); + goto unwind; + } + + priv = this->private; + + if (local->op_ret == -1) { + op_errno = local->op_errno; + goto unwind; + } + + STACK_WIND (frame, priv->is_quota_on? + quota_fallocate_cbk: default_fallocate_cbk, + FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len, + xdata); + return 0; + +unwind: + QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL); + return 0; +} + + int32_t quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, off_t offset, size_t len, dict_t *xdata) @@ -2732,6 +3224,7 @@ quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, quota_inode_ctx_t *ctx = NULL; quota_priv_t *priv = NULL; quota_dentry_t *dentry = NULL; + call_stub_t *stub = NULL; priv = this->private; GF_VALIDATE_OR_GOTO (this->name, priv, unwind); @@ -2750,7 +3243,8 @@ quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, frame->local = local; local->loc.inode = inode_ref (fd->inode); - ret = quota_inode_ctx_get (fd->inode, -1, -1, this, NULL, NULL, &ctx, 0); + ret = quota_inode_ctx_get (fd->inode, -1, -1, this, NULL, NULL, &ctx, + 0); if (ctx == NULL) { gf_log (this->name, GF_LOG_WARNING, "quota context not set in inode (gfid:%s)", @@ -2758,6 +3252,15 @@ quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, goto unwind; } + stub = fop_fallocate_stub(frame, quota_fallocate_helper, fd, mode, + offset, len, xdata); + if (stub == NULL) { + op_errno = ENOMEM; + goto unwind; + } + + priv = this->private; + GF_VALIDATE_OR_GOTO (this->name, priv, unwind); LOCK (&ctx->lock); { @@ -2773,27 +3276,26 @@ quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, * in ENOSPC errors attempting to allocate an already allocated range. */ local->delta = len; + local->stub = stub; local->link_count = parents; list_for_each_entry (dentry, &ctx->parents, next) { - ret = quota_check_limit (frame, fd->inode, this, dentry->name, - dentry->par); - if (ret == -1) { - op_errno = EDQUOT; - goto unwind; - } + quota_check_limit (frame, fd->inode, this, dentry->name, + dentry->par); } - -wind: - STACK_WIND (frame, priv->is_quota_on? - quota_fallocate_cbk: default_fallocate_cbk, - FIRST_CHILD(this), FIRST_CHILD(this)->fops->fallocate, fd, - mode, offset, len, xdata); return 0; unwind: QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL); return 0; + +wind: + STACK_WIND (frame, priv->is_quota_on? + quota_fallocate_cbk: default_fallocate_cbk, + FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len, + xdata); + return 0; } @@ -2853,108 +3355,6 @@ quota_forget (xlator_t *this, inode_t *inode) return 0; } - -int -quota_parse_limits (quota_priv_t *priv, xlator_t *this, dict_t *xl_options, - struct list_head *old_list) -{ - int32_t ret = -1; - char *str = NULL; - char *str_val = NULL; - char *path = NULL, *saveptr = NULL; - uint64_t value = 0; - limits_t *quota_lim = NULL, *old = NULL; - double soft_l = 0; - char *limit_dir = NULL; - char *saveptr_dir = NULL; - char *path_str = NULL; - - ret = dict_get_str (xl_options, "limit-set", &str); - - if (ret) { - gf_log (this->name, GF_LOG_INFO, "could not get the limits"); - /* limit may not be set at all on the volume yet */ - ret = 0; - goto err; - } - - path_str = gf_strdup (str); - if (!path_str) - goto err; - - - limit_dir = strtok_r (path_str, ",", &saveptr); - - while (limit_dir) { - QUOTA_ALLOC_OR_GOTO (quota_lim, limits_t, err); - saveptr_dir = NULL; - - path = strtok_r (limit_dir, ":", &saveptr_dir); - - str_val = strtok_r (NULL, ":", &saveptr_dir); - - ret = gf_string2bytesize (str_val, &value); - if (ret != 0) - goto err; - - quota_lim->hard_lim = value; - - str_val = strtok_r (NULL, ",", &saveptr_dir); - - soft_l = priv->default_soft_lim; - if (str_val) { - ret = gf_string2percent (str_val, &soft_l); - if (ret) - gf_log (this->name, GF_LOG_WARNING, - "Failed to convert str to " - "percent. Using default soft " - "limit"); - } - - quota_lim->soft_lim = soft_l; - - quota_lim->path = gf_strdup (path); - - gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64, - quota_lim->path, quota_lim->hard_lim); - - if (old_list != NULL) { - list_for_each_entry (old, old_list, - limit_list) { - if (strcmp (old->path, quota_lim->path) == 0) { - uuid_copy (quota_lim->gfid, - old->gfid); - break; - } - } - } - - LOCK (&priv->lock); - { - list_add_tail ("a_lim->limit_list, - &priv->limit_head); - } - UNLOCK (&priv->lock); - - limit_dir = strtok_r (NULL, ",", &saveptr); - } - - LOCK (&priv->lock); - { - list_for_each_entry (quota_lim, &priv->limit_head, limit_list) { - gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64, - quota_lim->path, quota_lim->hard_lim); - } - } - UNLOCK (&priv->lock); - - ret = 0; -err: - GF_FREE (path_str); - return ret; -} - - int32_t init (xlator_t *this) { @@ -2976,22 +3376,17 @@ init (xlator_t *this) QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err); - INIT_LIST_HEAD (&priv->limit_head); - LOCK_INIT (&priv->lock); this->private = priv; - ret = quota_parse_limits (priv, this, this->options, NULL); - - if (ret) { - goto err; - } - GF_OPTION_INIT ("deem-statfs", priv->consider_statfs, bool, err); GF_OPTION_INIT ("server-quota", priv->is_quota_on, bool, err); GF_OPTION_INIT ("default-soft-limit", priv->default_soft_lim, percent, err); + GF_OPTION_INIT ("soft-timeout", priv->soft_timeout, uint32, err); + GF_OPTION_INIT ("hard-timeout", priv->hard_timeout, uint32, err); + GF_OPTION_INIT ("volume-uuid", priv->volume_uuid, str, err); this->local_pool = mem_pool_new (quota_local_t, 64); if (!this->local_pool) { @@ -3001,76 +3396,23 @@ init (xlator_t *this) goto err; } + priv->rpc_clnt = quota_enforcer_init (this, this->options); + if (priv->rpc_clnt == NULL) { + ret = -1; + gf_log (this->name, GF_LOG_WARNING, + "rpc init failed"); + } + ret = 0; err: return ret; } - -void -__quota_reconfigure_inode_ctx (xlator_t *this, inode_t *inode, limits_t *limit) -{ - int ret = -1; - quota_inode_ctx_t *ctx = NULL; - - GF_VALIDATE_OR_GOTO ("quota", this, out); - GF_VALIDATE_OR_GOTO (this->name, inode, out); - GF_VALIDATE_OR_GOTO (this->name, limit, out); - - ret = quota_inode_ctx_get (inode, limit->hard_lim, limit->soft_lim, - this, NULL, NULL, &ctx, 1); - if ((ret == -1) || (ctx == NULL)) { - gf_log (this->name, GF_LOG_WARNING, "cannot create quota " - "context in inode(gfid:%s)", - uuid_utoa (inode->gfid)); - goto out; - } - - LOCK (&ctx->lock); - { - ctx->hard_lim = limit->hard_lim; - ctx->soft_lim = limit->soft_lim; - } - UNLOCK (&ctx->lock); - -out: - return; -} - - -void -__quota_reconfigure (xlator_t *this, inode_table_t *itable, limits_t *limit) -{ - inode_t *inode = NULL; - - if ((this == NULL) || (itable == NULL) || (limit == NULL)) { - goto out; - } - - if (!uuid_is_null (limit->gfid)) { - inode = inode_find (itable, limit->gfid); - } else { - inode = inode_resolve (itable, limit->path); - } - - if (inode != NULL) { - __quota_reconfigure_inode_ctx (this, inode, limit); - } - -out: - return; -} - - int reconfigure (xlator_t *this, dict_t *options) { int32_t ret = -1; quota_priv_t *priv = NULL; - limits_t *limit = NULL, *next = NULL, *new = NULL; - struct list_head head = {0, }; - xlator_t *top = NULL; - char found = 0; priv = this->private; @@ -3081,53 +3423,6 @@ reconfigure (xlator_t *this, dict_t *options) GF_OPTION_RECONF ("default-soft-limit", priv->default_soft_lim, options, percent, out); - INIT_LIST_HEAD (&head); - - LOCK (&priv->lock); - { - list_splice_init (&priv->limit_head, &head); - } - UNLOCK (&priv->lock); - - ret = quota_parse_limits (priv, this, options, &head); - if (ret == -1) { - gf_log ("quota", GF_LOG_WARNING, - "quota reconfigure failed, " - "new changes will not take effect"); - goto out; - } - - LOCK (&priv->lock); - { - top = ((glusterfs_ctx_t *)this->ctx)->active->top; - GF_ASSERT (top); - - list_for_each_entry (limit, &priv->limit_head, limit_list) { - __quota_reconfigure (this, top->itable, limit); - } - - list_for_each_entry_safe (limit, next, &head, limit_list) { - found = 0; - list_for_each_entry (new, &priv->limit_head, - limit_list) { - if (strcmp (new->path, limit->path) == 0) { - found = 1; - break; - } - } - - if (!found) { - limit->hard_lim = -1; - __quota_reconfigure (this, top->itable, limit); - } - - list_del_init (&limit->limit_list); - GF_FREE (limit); - } - } - UNLOCK (&priv->lock); - - ret = 0; out: @@ -3197,13 +3492,44 @@ struct volume_options options[] = { .min = 0, .max = LONG_MAX, }, - {.key = {"timeout"}, - .type = GF_OPTION_TYPE_SIZET, + {.key = {"soft-timeout"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, + .max = 60, + .default_value = "0", + .description = "quota caches the directory sizes on client. " + "soft-timeout indicates the timeout for the validity of" + " cache before soft-limit has been crossed." + }, + {.key = {"hard-timeout"}, + .type = GF_OPTION_TYPE_INT, .min = 0, .max = 60, .default_value = "0", - .description = "quota caches the directory sizes on client. Timeout " - "indicates the timeout for the cache to be revalidated." + .description = "quota caches the directory sizes on client. " + "hard-timeout indicates the timeout for the validity of" + " cache after soft-limit has been crossed." + }, + { .key = {"username"}, + .type = GF_OPTION_TYPE_ANY, + }, + { .key = {"password"}, + .type = GF_OPTION_TYPE_ANY, + }, + { .key = {"transport-type"}, + .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp", + "tcp/client", "ib-verbs/client", "rdma"}, + .type = GF_OPTION_TYPE_STR, + }, + { .key = {"remote-host"}, + .type = GF_OPTION_TYPE_INTERNET_ADDRESS, + }, + { .key = {"remote-port"}, + .type = GF_OPTION_TYPE_INT, + }, + { .key = {"volume-uuid"}, + .type = GF_OPTION_TYPE_STR, + .description = "uuid of the volume this brick is part of." }, {.key = {NULL}} }; diff --git a/xlators/features/quota/src/quota.h b/xlators/features/quota/src/quota.h index de9f6f16..84e7f917 100644 --- a/xlators/features/quota/src/quota.h +++ b/xlators/features/quota/src/quota.h @@ -12,12 +12,30 @@ #include "config.h" #endif +#ifndef _QUOTA_H +#define _QUOTA_H + #include "xlator.h" #include "call-stub.h" #include "defaults.h" -#include "byte-order.h" #include "common-utils.h" #include "quota-mem-types.h" +#include "glusterfs.h" +#include "compat.h" +#include "logging.h" +#include "dict.h" +#include "stack.h" +#include "common-utils.h" +#include "event.h" +#include "globals.h" +#include "rpcsvc.h" +#include "rpc-clnt.h" +#include "byte-order.h" +#include "glusterfs3-xdr.h" +#include "glusterfs3.h" +#include "xdr-generic.h" +#include "compat-errno.h" +#include "protocol-common.h" #define QUOTA_XATTR_PREFIX "trusted." #define DIRTY "dirty" @@ -27,6 +45,10 @@ #define READDIR_BUF 4096 #define QUOTA_UPDATE_USAGE_KEY "quota-update-usage" +#ifndef UUID_CANONICAL_FORM_LEN +#define UUID_CANONICAL_FORM_LEN 36 +#endif + #define WIND_IF_QUOTAOFF(is_quota_on, label) \ if (!is_quota_on) \ goto label; @@ -125,63 +147,45 @@ struct quota_inode_ctx { typedef struct quota_inode_ctx quota_inode_ctx_t; struct quota_local { - gf_lock_t lock; - uint32_t validate_count; - uint32_t link_count; - loc_t loc; - loc_t oldloc; - loc_t newloc; - loc_t validate_loc; - int64_t delta; - int32_t op_ret; - int32_t op_errno; - int64_t size; - int64_t hard_lim; - int64_t soft_lim; - char just_validated; - inode_t *inode; - call_stub_t *stub; + gf_lock_t lock; + uint32_t validate_count; + uint32_t link_count; + loc_t loc; + loc_t oldloc; + loc_t newloc; + loc_t validate_loc; + int64_t delta; + int32_t op_ret; + int32_t op_errno; + int64_t size; + char just_validated; + fop_lookup_cbk_t validate_cbk; + inode_t *inode; + call_stub_t *stub; + struct iobref *iobref; }; -typedef struct quota_local quota_local_t; - - -struct qd_vols_conf { - char *name; - inode_table_t *itable; - uint32_t log_timeout; - gf_boolean_t threads_status; - double default_soft_lim; - gf_lock_t lock; - loc_t root_loc; - uint32_t soft_timeout; - uint32_t hard_timeout; - struct list_head limit_head; - call_frame_t *frame; -}; -typedef struct qd_vols_conf qd_vols_conf_t; - +typedef struct quota_local quota_local_t; struct quota_priv { - int64_t timeout; - double default_soft_lim; - gf_boolean_t is_quota_on; - gf_boolean_t consider_statfs; - struct list_head limit_head; - qd_vols_conf_t **qd_vols_conf; - gf_lock_t lock; -}; -typedef struct quota_priv quota_priv_t; - - -struct limits { - struct list_head limit_list; - char *path; - uuid_t gfid; - int64_t prev_size; - struct timeval prev_log_tv; - int64_t hard_lim; - int64_t soft_lim; - struct timeval expire; - uint32_t timeout; + uint32_t soft_timeout; + uint32_t hard_timeout; + double default_soft_lim; + gf_boolean_t is_quota_on; + gf_boolean_t consider_statfs; + gf_lock_t lock; + rpc_clnt_prog_t *quota_enforcer; + struct rpcsvc_program *quotad_aggregator; + struct rpc_clnt *rpc_clnt; + rpcsvc_t *rpcsvc; + inode_table_t *itable; + char *volume_uuid; }; -typedef struct limits limits_t; +typedef struct quota_priv quota_priv_t; + +int +quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, + dict_t *xdata, fop_lookup_cbk_t cbk); +struct rpc_clnt * +quota_enforcer_init (xlator_t *this, dict_t *options); + +#endif diff --git a/xlators/features/quota/src/quotad-aggregator.c b/xlators/features/quota/src/quotad-aggregator.c new file mode 100644 index 00000000..0363135f --- /dev/null +++ b/xlators/features/quota/src/quotad-aggregator.c @@ -0,0 +1,281 @@ +/* + Copyright (c) 2008-2012 Red Hat, Inc. + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "quota.h" +#include "quotad-helpers.h" +#include "quotad-aggregator.h" + +struct rpcsvc_program quotad_aggregator_prog; + +struct iobuf * +quotad_serialize_reply (rpcsvc_request_t *req, void *arg, struct iovec *outmsg, + xdrproc_t xdrproc) +{ + struct iobuf *iob = NULL; + ssize_t retlen = 0; + ssize_t xdr_size = 0; + + GF_VALIDATE_OR_GOTO ("server", req, ret); + + /* First, get the io buffer into which the reply in arg will + * be serialized. + */ + if (arg && xdrproc) { + xdr_size = xdr_sizeof (xdrproc, arg); + iob = iobuf_get2 (req->svc->ctx->iobuf_pool, xdr_size); + if (!iob) { + gf_log_callingfn (THIS->name, GF_LOG_ERROR, + "Failed to get iobuf"); + goto ret; + }; + + iobuf_to_iovec (iob, outmsg); + /* Use the given serializer to translate the give C structure in arg + * to XDR format which will be written into the buffer in outmsg. + */ + /* retlen is used to received the error since size_t is unsigned and we + * need -1 for error notification during encoding. + */ + + retlen = xdr_serialize_generic (*outmsg, arg, xdrproc); + if (retlen == -1) { + /* Failed to Encode 'GlusterFS' msg in RPC is not exactly + failure of RPC return values.. client should get + notified about this, so there are no missing frames */ + gf_log_callingfn ("", GF_LOG_ERROR, "Failed to encode message"); + req->rpc_err = GARBAGE_ARGS; + retlen = 0; + } + } + outmsg->iov_len = retlen; +ret: + if (retlen == -1) { + iobuf_unref (iob); + iob = NULL; + } + + return iob; +} + +int +quotad_aggregator_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, + void *arg, struct iovec *payload, + int payloadcount, struct iobref *iobref, + xdrproc_t xdrproc) +{ + struct iobuf *iob = NULL; + int ret = -1; + struct iovec rsp = {0,}; + quotad_aggregator_state_t *state = NULL; + char new_iobref = 0; + + GF_VALIDATE_OR_GOTO ("server", req, ret); + + if (frame) { + state = frame->root->state; + frame->local = NULL; + } + + if (!iobref) { + iobref = iobref_new (); + if (!iobref) { + goto ret; + } + + new_iobref = 1; + } + + iob = quotad_serialize_reply (req, arg, &rsp, xdrproc); + if (!iob) { + gf_log ("", GF_LOG_ERROR, "Failed to serialize reply"); + goto ret; + } + + iobref_add (iobref, iob); + + ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount, + iobref); + + iobuf_unref (iob); + + ret = 0; +ret: + if (state) { + quotad_aggregator_free_state (state); + } + + if (frame) { + if (frame->root->trans) + rpc_transport_unref (frame->root->trans); + + STACK_DESTROY (frame->root); + } + + if (new_iobref) { + iobref_unref (iobref); + } + + return ret; +} + +int +quotad_aggregator_lookup_cbk (xlator_t *this, call_frame_t *frame, + struct gfs3_lookup_rsp *rsp) +{ + quotad_aggregator_submit_reply (frame, frame->local, rsp, NULL, 0, NULL, + (xdrproc_t)xdr_gfs3_lookup_rsp); + + return 0; +} + +int +quotad_aggregator_lookup (rpcsvc_request_t *req) +{ + call_frame_t *frame = NULL; + gfs3_lookup_req args = {{0,},}; + int ret = -1, op_errno = 0; + gfs3_lookup_rsp rsp = {0,}; + quotad_aggregator_state_t *state = NULL; + xlator_t *this = NULL; + + GF_VALIDATE_OR_GOTO ("quotad-aggregator", req, err); + + this = THIS; + + args.bname = alloca (req->msg[0].iov_len); + args.xdata.xdata_val = alloca (req->msg[0].iov_len); + + ret = xdr_to_generic (req->msg[0], &args, + (xdrproc_t)xdr_gfs3_lookup_req); + if (ret < 0) { + rsp.op_errno = EINVAL; + goto err; + } + + frame = quotad_aggregator_get_frame_from_req (req); + if (frame == NULL) { + rsp.op_errno = ENOMEM; + goto err; + } + + state = frame->root->state; + + GF_PROTOCOL_DICT_UNSERIALIZE (this, state->xdata, + (args.xdata.xdata_val), + (args.xdata.xdata_len), ret, + op_errno, err); + + + ret = qd_nameless_lookup (this, frame, &args, state->xdata, + quotad_aggregator_lookup_cbk); + if (ret) { + rsp.op_errno = ret; + goto err; + } + + return ret; + +err: + rsp.op_ret = -1; + rsp.op_errno = op_errno; + + quotad_aggregator_lookup_cbk (this, frame, &rsp); + return ret; +} + +int +quotad_aggregator_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, + void *data) +{ + if (!xl || !data) { + gf_log_callingfn ("server", GF_LOG_WARNING, + "Calling rpc_notify without initializing"); + goto out; + } + + switch (event) { + case RPCSVC_EVENT_ACCEPT: + break; + + case RPCSVC_EVENT_DISCONNECT: + break; + + default: + break; + } + +out: + return 0; +} + +int +quotad_aggregator_init (xlator_t *this) +{ + quota_priv_t *priv = NULL; + int ret = -1; + + priv = this->private; + + dict_set_str (this->options, "transport.address-family", "unix"); + dict_set_str (this->options, "transport-type", "socket"); + dict_set_str (this->options, "transport.socket.listen-path", + "/tmp/quotad.socket"); + + /* RPC related */ + priv->rpcsvc = rpcsvc_init (this, this->ctx, this->options, 0); + if (priv->rpcsvc == NULL) { + gf_log (this->name, GF_LOG_WARNING, + "creation of rpcsvc failed"); + ret = -1; + goto out; + } + + ret = rpcsvc_create_listeners (priv->rpcsvc, this->options, + this->name); + if (ret < 1) { + gf_log (this->name, GF_LOG_WARNING, + "creation of listener failed"); + ret = -1; + goto out; + } + + priv->quotad_aggregator = "ad_aggregator_prog; + quotad_aggregator_prog.options = this->options; + + ret = rpcsvc_program_register (priv->rpcsvc, "ad_aggregator_prog); + if (ret) { + gf_log (this->name, GF_LOG_WARNING, + "registration of program (name:%s, prognum:%d, " + "progver:%d) failed", quotad_aggregator_prog.progname, + quotad_aggregator_prog.prognum, + quotad_aggregator_prog.progver); + goto out; + } + + ret = 0; +out: + return ret; +} + +rpcsvc_actor_t quotad_aggregator_actors[] = { + [GF_AGGREGATOR_NULL] = {"NULL", GF_AGGREGATOR_NULL, NULL, NULL, 0, + DRC_NA}, + [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", GF_AGGREGATOR_NULL, + quotad_aggregator_lookup, NULL, 0, DRC_NA}, +}; + + +struct rpcsvc_program quotad_aggregator_prog = { + .progname = "GlusterFS 3.3", + .prognum = GLUSTER_AGGREGATOR_PROGRAM, + .progver = GLUSTER_AGGREGATOR_VERSION, + .numactors = GF_AGGREGATOR_MAXVALUE, + .actors = quotad_aggregator_actors +}; diff --git a/xlators/features/quota/src/quotad-aggregator.h b/xlators/features/quota/src/quotad-aggregator.h new file mode 100644 index 00000000..ac5eb76b --- /dev/null +++ b/xlators/features/quota/src/quotad-aggregator.h @@ -0,0 +1,37 @@ +/* + Copyright (c) 2008-2012 Red Hat, Inc. + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _QUOTAD_AGGREGATOR_H +#define _QUOTAD_AGGREGATOR_H + +#include "quota.h" +#include "stack.h" +#include "glusterfs3-xdr.h" +#include "inode.h" + +typedef struct { + void *pool; + xlator_t *this; + xlator_t *active_subvol; + inode_table_t *itable; + loc_t loc; + dict_t *xdata; +} quotad_aggregator_state_t; + +typedef int (*quotad_aggregator_lookup_cbk_t) (xlator_t *this, + call_frame_t *frame, + struct gfs3_lookup_rsp *rsp); +int +qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req, + dict_t *xdata, quotad_aggregator_lookup_cbk_t lookup_cbk); +int +quotad_aggregator_init (xlator_t *this); + +#endif diff --git a/xlators/features/quota/src/quotad-helpers.c b/xlators/features/quota/src/quotad-helpers.c new file mode 100644 index 00000000..f684ced0 --- /dev/null +++ b/xlators/features/quota/src/quotad-helpers.c @@ -0,0 +1,107 @@ +/* + Copyright (c) 2008-2012 Red Hat, Inc. + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "quotad-helpers.h" + +quotad_aggregator_state_t * +get_quotad_aggregator_state (xlator_t *this, rpcsvc_request_t *req) +{ + quotad_aggregator_state_t *state = NULL; + xlator_t *active_subvol = NULL; + quota_priv_t *priv = NULL; + + state = (void *)GF_CALLOC (1, sizeof (*state), + gf_quota_mt_aggregator_state_t); + if (!state) + return NULL; + + state->this = THIS; + priv = this->private; + + LOCK (&priv->lock); + { + active_subvol = state->active_subvol = FIRST_CHILD (this); + } + UNLOCK (&priv->lock); + + if (active_subvol->itable == NULL) + active_subvol->itable = inode_table_new (4096, active_subvol); + + state->itable = active_subvol->itable; + + state->pool = this->ctx->pool; + + return state; +} + +void +quotad_aggregator_free_state (quotad_aggregator_state_t *state) +{ + if (state->xdata) + dict_unref (state->xdata); + + GF_FREE (state); +} + +call_frame_t * +quotad_aggregator_alloc_frame (rpcsvc_request_t *req) +{ + call_frame_t *frame = NULL; + quotad_aggregator_state_t *state = NULL; + xlator_t *this = NULL; + + GF_VALIDATE_OR_GOTO ("server", req, out); + GF_VALIDATE_OR_GOTO ("server", req->trans, out); + GF_VALIDATE_OR_GOTO ("server", req->svc, out); + GF_VALIDATE_OR_GOTO ("server", req->svc->ctx, out); + + this = req->svc->mydata; + + frame = create_frame (this, req->svc->ctx->pool); + if (!frame) + goto out; + + state = get_quotad_aggregator_state (this, req); + if (!state) + goto out; + + frame->root->state = state; + frame->root->unique = 0; + + frame->this = this; +out: + return frame; +} + +call_frame_t * +quotad_aggregator_get_frame_from_req (rpcsvc_request_t *req) +{ + call_frame_t *frame = NULL; + + GF_VALIDATE_OR_GOTO ("server", req, out); + + frame = quotad_aggregator_alloc_frame (req); + if (!frame) + goto out; + + frame->root->op = req->procnum; + + frame->root->unique = req->xid; + + frame->root->uid = req->uid; + frame->root->gid = req->gid; + frame->root->pid = req->pid; + frame->root->trans = rpc_transport_ref (req->trans); + frame->root->lk_owner = req->lk_owner; + + frame->local = req; +out: + return frame; +} diff --git a/xlators/features/quota/src/quotad-helpers.h b/xlators/features/quota/src/quotad-helpers.h new file mode 100644 index 00000000..a10fb7fa --- /dev/null +++ b/xlators/features/quota/src/quotad-helpers.h @@ -0,0 +1,24 @@ +/* + Copyright (c) 2008-2012 Red Hat, Inc. + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef QUOTAD_HELPERS_H +#define QUOTAD_HELPERS_H + +#include "rpcsvc.h" +#include "quota.h" +#include "quotad-aggregator.h" + +void +quotad_aggregator_free_state (quotad_aggregator_state_t *state); + +call_frame_t * +quotad_aggregator_get_frame_from_req (rpcsvc_request_t *req); + +#endif diff --git a/xlators/features/quota/src/quotad.c b/xlators/features/quota/src/quotad.c index a5e51add..d6bf5f0e 100644 --- a/xlators/features/quota/src/quotad.c +++ b/xlators/features/quota/src/quotad.c @@ -7,77 +7,9 @@ later), or the GNU General Public License, version 2 (GPLv2), in all cases as published by the Free Software Foundation. */ -#include - #include "quota.h" +#include "quotad-aggregator.h" #include "common-utils.h" -#include "defaults.h" -#include "syncop.h" -#include "libgen.h" -#include "timer.h" - - -inline uint64_t -quota_time_elapsed (struct timeval *now, struct timeval *then) -{ - return (now->tv_sec - then->tv_sec); -} - - -int32_t -quota_timeout (struct timeval *tv, uint64_t timeout) -{ - struct timeval now = {0,}; - int32_t timed_out = 0; - - gettimeofday (&now, NULL); - - if (quota_time_elapsed (&now, tv) >= timeout) { - timed_out = 1; - } - - return timed_out; -} - -/* Returns itable->root, also creates itable if not present */ -inode_t * -qd_build_root_inode (xlator_t *this, qd_vols_conf_t *this_vol) -{ - if (!this_vol->itable) { - this_vol->itable = inode_table_new (0, this); - if (!this_vol->itable) - return NULL; - } - - return inode_ref (this_vol->itable->root); -} - -int -qd_resolve_root (xlator_t *subvol, loc_t *root_loc, - struct iatt *iatt, dict_t *dict_req, dict_t **dict_rsp) -{ - int ret = -1; - - ret = syncop_lookup (subvol, root_loc, dict_req, iatt, dict_rsp, NULL); - if (-1 == ret) { - gf_log (subvol->name, GF_LOG_ERROR, "Received %s for lookup " - "on / (vol:%s)", strerror (errno), subvol->name); - } - return ret; -} - -int -qd_build_root_loc (xlator_t *this, xlator_t *subvol, inode_t *inode, loc_t *loc) -{ - - loc->path = gf_strdup ("/"); - loc->inode = inode; - uuid_clear (loc->gfid); - loc->gfid[15] = 1; - - return qd_resolve_root (subvol, loc, NULL, NULL, NULL); -} - int32_t mem_acct_init (xlator_t *this) @@ -98,637 +30,106 @@ mem_acct_init (xlator_t *this) return ret; } -/** - * Takes the limit string, parse it, fill limits_t struct and insert into - * above-soft list. - * - * Format for limit string - * = [,]* - * = - * :[:] - */ -int -qd_parse_limits (quota_priv_t *priv, xlator_t *this, char *limit_str, - struct list_head *old_list, qd_vols_conf_t *this_vol) +int32_t +qd_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, struct iatt *postparent) { - int32_t ret = -1; - char *str_val = NULL; - char *path = NULL, *saveptr = NULL; - uint64_t value = 0; - limits_t *quota_lim = NULL; - char *str = NULL; - double soft_l = -1; - char *limit_on_dir = NULL; - char *saveptr_dir = NULL; - - str = gf_strdup (limit_str); - - if (str) { - limit_on_dir = strtok_r (str, ",", &saveptr); - - while (limit_on_dir) { - QUOTA_ALLOC_OR_GOTO (quota_lim, limits_t, err); - gettimeofday ("a_lim->prev_log_tv, NULL); - - saveptr_dir = NULL; - - path = strtok_r (limit_on_dir, ":", &saveptr_dir); - - str_val = strtok_r (NULL, ":", &saveptr_dir); - - ret = gf_string2bytesize (str_val, &value); - if (0 != ret) - goto err; - - quota_lim->hard_lim = value; - - str_val = strtok_r (NULL, ",", &saveptr_dir); - - soft_l = this_vol->default_soft_lim; - if (str_val) { - ret = gf_string2percent (str_val, &soft_l); - if (ret) - gf_log (this->name, GF_LOG_WARNING, - "Failed to convert str to " - "percent. Using default soft " - "limit"); - } - - quota_lim->soft_lim = (int64_t) quota_lim->hard_lim * - (soft_l / 100); - - quota_lim->path = gf_strdup (path); - if (!quota_lim->path) { - gf_log (this->name, GF_LOG_ERROR, "ENOMEM " - "Received copying the path"); - goto err; - } - - quota_lim->prev_size = -1; - - gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64":%"PRId32, - quota_lim->path, quota_lim->hard_lim, - (int)quota_lim->soft_lim); - - /* This is used in the reconfigure path, so not used - * by quotad as of now. - if (NULL != old_list) { - list_for_each_entry (old, old_list, - limit_list) { - if (0 == - strcmp (old->path, quota_lim->path)) { - uuid_copy (quota_lim->gfid, - old->gfid); - break; - } - } - } */ - - LOCK (&this_vol->lock); - { - list_add_tail ("a_lim->limit_list, - &this_vol->limit_head); - } - UNLOCK (&this_vol->lock); - - limit_on_dir = strtok_r (NULL, ",", &saveptr); - } - } else { - gf_log (this->name, GF_LOG_INFO, - "no \"limit-set\" option provided"); - } - - ret = 0; -err: - GF_FREE (str); - return ret; -} + quotad_aggregator_lookup_cbk_t lookup_cbk = NULL; + gfs3_lookup_rsp rsp = {0, }; + lookup_cbk = cookie; -xlator_t * -qd_get_subvol (xlator_t *this, qd_vols_conf_t *this_vol) -{ - xlator_list_t *subvol = NULL; + rsp.op_ret = op_ret; + rsp.op_errno = op_errno; - for (subvol = this->children; subvol; subvol = subvol->next) - if (0 == strcmp (subvol->xlator->name, this_vol->name)) - return subvol->xlator; + gf_stat_from_iatt (&rsp.postparent, postparent); - return NULL; -} + GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val), + rsp.xdata.xdata_len, rsp.op_errno, out); -/* Logs if - * i. Usage crossed soft limit - * ii. Usage above soft limit and alert-time timed out - */ -void -qd_log_usage (xlator_t *this, qd_vols_conf_t *conf, limits_t *entry, - int64_t cur_size) -{ - struct timeval cur_time = {0,}; - char *usage_str = NULL; - - gettimeofday (&cur_time, NULL); - - /* Usage crossed/reached soft limit? */ - if (DID_REACH_LIMIT (entry->soft_lim, entry->prev_size, cur_size)) { - entry->prev_log_tv = cur_time; - usage_str = gf_uint64_2human_readable (entry->soft_lim); - gf_log (this->name, GF_LOG_ALERT, "Usage crossed soft limit: " - "%s for %s (vol:%s)", usage_str, entry->path, - conf->name); - } - /* Usage crossed/reached hard limit? */ - else if (DID_REACH_LIMIT (entry->hard_lim, entry->prev_size, - cur_size)) { - entry->prev_log_tv = cur_time; - usage_str = gf_uint64_2human_readable (entry->hard_lim); - gf_log (this->name, GF_LOG_ALERT, "Usage reached hard limit: " - "%s for %s (vol:%s)", usage_str, entry->path, - conf->name); - } - /* Usage is above soft limit and 'alert-time' timed out */ - else if (cur_size > entry->soft_lim && - quota_timeout (&entry->prev_log_tv, conf->log_timeout)) { - entry->prev_log_tv = cur_time; - usage_str = gf_uint64_2human_readable (cur_size); - gf_log (this->name, GF_LOG_ALERT, "Usage %s %s limit for %s " - "(vol:%s)", usage_str, (cur_size < entry->hard_lim)? - "is above soft": "has reached hard", entry->path, - conf->name); - } - if (usage_str) - GF_FREE (usage_str); -} + gf_stat_from_iatt (&rsp.stat, buf); -int -qd_build_child_loc (loc_t *child, loc_t *parent, char *name) -{ - if (!child) { - goto err; - } - - if (strcmp (parent->path, "/") == 0) - gf_asprintf ((char **)&child->path, "/%s", name); - else - gf_asprintf ((char **)&child->path, "%s/%s", parent->path, - name); - - if (!child->path) { - goto err; - } - - child->name = strrchr (child->path, '/'); - if (child->name) - child->name++; +out: + lookup_cbk (this, frame, &rsp); - child->parent = inode_ref (parent->inode); - if (!child->inode) - child->inode = inode_new (parent->inode->table); + GF_FREE (rsp.xdata.xdata_val); - if (!child->inode) { - goto err; - } - if (!uuid_is_null(parent->gfid)) - uuid_copy (child->pargfid, parent->gfid); - else - uuid_copy (child->pargfid, parent->inode->gfid); + inode_unref (inode); return 0; -err: - loc_wipe (child); - return -1; } -#define QUOTA_IDLE_TIMEOUT 30 -int -qd_check_enforce (qd_vols_conf_t *conf, dict_t *dict, limits_t *entry, - loc_t *entry_loc, xlator_t *subvol) +xlator_t * +qd_find_subvol (xlator_t *this, char *volume_uuid) { - xlator_t *this = THIS; - int64_t *size = NULL; - int ret = -1; - int64_t cur_size = 0; - int64_t tmp_size = 0; - dict_t *setxattr_dict = NULL; - int64_t *quota_xlator_size = NULL; - - ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **)&size); - if (ret) { - gf_log (this->name, GF_LOG_WARNING, "Couldn't get size" - " from the dict (%s)", entry->path); - goto out; - } + xlator_list_t *child = NULL; + xlator_t *subvol = NULL; - cur_size = ntoh64 (*size); - - qd_log_usage (this, conf, entry, cur_size); - - /* if size hasn't changed, skip the update and bump the timeout */ - if (entry->prev_size == cur_size) { - /* Because of dht we might be getting the correct aggregated - size. But the quota xlator in the server graph might not - be having the right value. So get the size that quota xlator - is maintaining in its context. If it is different than the - total aggregated size, then update the quota xlator with - the correct size through setxattr. - */ - ret = dict_get_bin (dict, "trusted.limit.set", - (void **)"a_xlator_size); - if (ret) - gf_log (this->name, GF_LOG_WARNING, "Couldn't get size" - " from the dict (%s)", entry->path); - else - tmp_size = ntoh64 (*quota_xlator_size); - - if (entry->prev_size == tmp_size) { - entry->timeout = min((entry->timeout << 1), - QUOTA_IDLE_TIMEOUT); - goto out; - } - } - - QUOTA_ALLOC_OR_GOTO (size, int64_t, out); - *size = hton64 (cur_size); - - setxattr_dict = dict_new(); - ret = dict_set_bin (setxattr_dict, QUOTA_UPDATE_USAGE_KEY, size, - sizeof (int64_t)); - if (-1 == ret) { - gf_log (this->name, GF_LOG_WARNING, - "Couldn't set dict"); + if (!this || !volume_uuid) goto out; - } - - - /* There is a possibility that, after a setxattr is done, - a rename might happen and the resolve will fail again. - */ - ret = syncop_setxattr (subvol, entry_loc, setxattr_dict, 0); - if (ret) { - if (errno == ESTALE || errno == ENOENT) - inode_forget (entry_loc->inode, 0); - gf_log (this->name, GF_LOG_ERROR, - "Received ERROR:%s in updating quota value %s " - " (vol:%s). Quota enforcement may not be" - " accurate", strerror (errno), entry->path, - subvol->name); - goto out; - } - LOCK(&conf->lock); - /* set the timeout based on usage */ - if (cur_size < entry->soft_lim) - entry->timeout = conf->soft_timeout; - else - entry->timeout = conf->hard_timeout; - - entry->prev_size = cur_size; - UNLOCK(&conf->lock); - -out: - if (setxattr_dict) - dict_unref (setxattr_dict); - - return ret; -} - -int -qd_resolve_handle_error (loc_t *comp_loc, inode_t *inode, - int op_ret, int op_errno) -{ - - if (op_ret) { - switch (op_errno) { - case ESTALE: - case ENOENT: - if (comp_loc->inode) - inode_forget (comp_loc->inode, 0); - break; - default: - gf_log ("", GF_LOG_ERROR, "lookup on %s returned %s", - comp_loc->path, strerror(op_errno)); + for (child = this->children; child; child = child->next) { + if (strncmp (child->xlator->name, volume_uuid, + strlen (child->xlator->name)) == 0) { + subvol = child->xlator; + break; } } - return 0; -} - -int -qd_resolve_entry (qd_vols_conf_t *conf, xlator_t *subvol, limits_t *entry, - dict_t *dict_req, loc_t *loc, dict_t **dict_rsp, - int force) -{ - char *component = NULL; - char *next_comp = NULL; - char *saveptr = NULL; - char *path = NULL; - int ret = -1; - struct iatt iatt = {0,}; - inode_t *inode = NULL; - dict_t *tmp_dict = NULL; - loc_t comp_loc = {0,}; - loc_t par_loc = {0,}; - int need_lookup = 0; - - path = gf_strdup (entry->path); - - loc_copy (&par_loc, &conf->root_loc); - for (component = strtok_r (path, "/", &saveptr); - component; component = next_comp) { - - next_comp = strtok_r (NULL, "/", &saveptr); - - inode = inode_grep (conf->itable, - par_loc.inode, - component); - - /* No need to forget and unref the inode if revalidate flag - is set (i.e @force). Reason: - Just because we want to revalidate, does not mean - the inode is not valid. For invalidating the inode, - (i.e forgetting it) let lookup be sent, and the decision - should be made depending upon lookup's reply. - */ - - /* if inode is found, then skip lookup unless last component */ - if (inode) { - comp_loc.inode = inode; - need_lookup = 0; - } else { - need_lookup = 1; - } - - qd_build_child_loc (&comp_loc, &par_loc, component); - /* Get the xattrs in lookup for the last component */ - if (!next_comp) { - tmp_dict = dict_req; - need_lookup = 1; - } - - if (need_lookup || force) { - ret = syncop_lookup (subvol, &comp_loc, tmp_dict, &iatt, - dict_rsp, NULL); - if (ret) { - /* invalidate inode got from inode_grep if - * ESTALE/ENOENT */ - qd_resolve_handle_error (&comp_loc, inode, - ret, errno); - goto out; - } - - if (!IA_ISDIR (iatt.ia_type)) { - gf_log (subvol->name, GF_LOG_ERROR, - "%s is not a directory", - comp_loc.path); - goto out; - } - - inode = inode_link (comp_loc.inode, par_loc.inode, - component, &iatt); - comp_loc.inode = inode; - inode_lookup (comp_loc.inode); - inode_unref (inode); - } - inode = NULL; - loc_wipe (&par_loc); - loc_copy (&par_loc, &comp_loc); - loc_wipe (&comp_loc); - } - ret = 0; - loc_copy (loc, &par_loc); - out: - GF_FREE(path); - loc_wipe (&par_loc); - if (ret) - loc_wipe (&comp_loc); - return ret; + return subvol; } int -qd_handle_entry (qd_vols_conf_t *conf, xlator_t *subvol, limits_t *entry, - dict_t *dict_req, int revalidate) +qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req, + dict_t *xdata, quotad_aggregator_lookup_cbk_t lookup_cbk) { - dict_t *dict_rsp = NULL; - loc_t entry_loc = {0,}; - int ret = -1; - - if (!strcmp (entry->path, "/")) { - ret = qd_resolve_root (subvol, &conf->root_loc, NULL, dict_req, - &dict_rsp); - if (ret) { - gf_log (subvol->name, GF_LOG_ERROR, "lookup on / " - "(%s)", strerror(errno)); - goto err; - } - loc_copy (&entry_loc, &conf->root_loc); - - } else { - ret = qd_resolve_entry (conf, subvol, entry, dict_req, - &entry_loc, &dict_rsp, revalidate); - /* if resolve failed, force resolve from "/" once */ - if (ret) { - if (errno == ENOENT) - entry->timeout = QUOTA_IDLE_TIMEOUT; - else - gf_log (subvol->name, GF_LOG_ERROR, - "Quota check on %s failed ERR:%s", - entry->path, strerror (errno)); - if (!revalidate) { - ret = qd_handle_entry (conf, subvol, entry, - dict_req, 1); - if (ret) - goto err; - } - goto err; - } - } - ret = qd_check_enforce (conf, dict_rsp, entry, &entry_loc, subvol); - if (ret) - gf_log (subvol->name, GF_LOG_ERROR, - "Failed to enforce quota on %s", entry_loc.path); -err: - loc_wipe (&entry_loc); - if (dict_rsp) - dict_unref (dict_rsp); - return 0; -} + gfs3_lookup_rsp rsp = {0, }; + int op_errno = 0, ret = -1; + loc_t loc = {0, }; + quotad_aggregator_state_t *state = NULL; + quota_priv_t *priv = NULL; + xlator_t *subvol = NULL; + char *volume_uuid = NULL; -int -qd_iterator (qd_vols_conf_t *conf, xlator_t *subvol) -{ - limits_t *entry = NULL; - limits_t *next = NULL; - int32_t ret; - dict_t *dict_req = NULL; - struct timeval now; - struct timeval next_expire = {0,}; - - GF_VALIDATE_OR_GOTO ("qd-iterator", conf, out); - GF_VALIDATE_OR_GOTO ("qd-iterator", subvol, out); - - dict_req = dict_new (); - GF_VALIDATE_OR_GOTO ("qd-iterator", dict_req, out); - ret = dict_set_uint64 (dict_req, QUOTA_SIZE_KEY, 0); - if (ret) { - gf_log ("qd-iterator", GF_LOG_ERROR, "dict set failed for " - "QUOTA SIZE key"); - goto out; - } - - gettimeofday(&now, NULL); - list_for_each_entry_safe (entry, next, &conf->limit_head, limit_list) { - /* just use the same start time across the entire iteration */ - if (timercmp(&now, &entry->expire, <)) - goto check_next_expire; + priv = this->private; + state = frame->root->state; - ret = qd_handle_entry (conf, subvol, entry, dict_req, 0); - if (ret) { - gf_log ("qd-iterator", GF_LOG_ERROR, "Failed to check " - "quota limit on %s", entry->path); - } + frame->root->op = GF_FOP_LOOKUP; - gettimeofday(&entry->expire, NULL); - entry->expire.tv_sec += entry->timeout; - gf_log("qd-iterator", GF_LOG_TRACE, - "volume %s path %s usage %lu (hard %lu soft %lu) expire in %ds", - conf->name, entry->path, entry->prev_size, - entry->hard_lim, entry->soft_lim, entry->timeout); - -check_next_expire: - if (!next_expire.tv_sec || - timercmp(&entry->expire, &next_expire, <)) - next_expire = entry->expire; + loc.inode = inode_new (state->itable); + if (loc.inode == NULL) { + op_errno = ENOMEM; + goto out; } - ret = 0; - /* return a hint as to when we'll next have something to do */ - gettimeofday(&now, NULL); - if (timercmp(&now, &next_expire, <)) { - timersub(&next_expire, &now, &next_expire); - ret = next_expire.tv_sec; - if (next_expire.tv_usec) - ret++; - } -out: - if (dict_req) - dict_unref (dict_req); - - return ret; -} + memcpy (loc.gfid, req->gfid, 16); -int -qd_trigger_periodically (void *args) -{ - int ret = -1; - xlator_t *this = NULL; - xlator_t *subvol = NULL; - inode_t *root_inode = NULL; - qd_vols_conf_t *conf = NULL; - - this = THIS; - conf = args; - - subvol = qd_get_subvol (this, conf); - if (!subvol) { - gf_log (this->name, GF_LOG_ERROR, "No subvol found"); - return -1; + ret = dict_get_str (xdata, "volume-uuid", &volume_uuid); + if (ret < 0) { + op_errno = EINVAL; + goto out; } - if (!conf->root_loc.path) { - root_inode = qd_build_root_inode (this, conf); - if (!root_inode) { - gf_log (this->name, GF_LOG_ERROR, - "New itable creation failed"); - return -1; - } - - ret = qd_build_root_loc (this, subvol, root_inode, - &conf->root_loc); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "Failed to build root_loc for %s", conf->name); - return -1; - } + subvol = qd_find_subvol (this, volume_uuid); + if (subvol == NULL) { + op_errno = EINVAL; + goto out; } - if (list_empty(&conf->limit_head)) - return QUOTA_IDLE_TIMEOUT; - - ret = qd_iterator (conf, subvol); - if (ret < 0) - gf_log (this->name, GF_LOG_WARNING, - "Couldn't update the usage, frequent " - "log may lead usage to cross beyond " - "limit"); - - return ret; -} - -static void create_iter_task(void *); - -int -qd_trigger_periodically_cbk(int ret, call_frame_t *frame, void *args) -{ - qd_vols_conf_t *conf = args; - struct timeval delta = { 0, }; - - if (ret < 0) - gf_log ("quotad", GF_LOG_ERROR, - "Synctask stopped unexpectedly, trying to restart"); - else - delta.tv_sec = ret; - - conf->frame = frame; - gf_timer_call_after(THIS->ctx, delta, create_iter_task, conf); - - return ret; -} - -static void -create_iter_task(void *data) -{ - qd_vols_conf_t *conf = data; - int ret; - - ret = synctask_new(THIS->ctx->env, qd_trigger_periodically, - qd_trigger_periodically_cbk, conf->frame, - conf); - if (ret < 0) - gf_log("quotad", GF_LOG_ERROR, - "Synctask creation failed for %s", conf->name); -} - -int -qd_start_threads (xlator_t *this, int subvol_idx) -{ - quota_priv_t *priv = NULL; - int ret = 0; - qd_vols_conf_t *this_vol = NULL; - - priv = this->private; + STACK_WIND_COOKIE (frame, qd_lookup_cbk, lookup_cbk, subvol, + subvol->fops->lookup, &loc, xdata); + return 0; - this_vol = priv->qd_vols_conf[subvol_idx]; +out: + rsp.op_ret = -1; + rsp.op_errno = op_errno; - if (list_empty (&this_vol->limit_head)) { - gf_log (this->name, GF_LOG_DEBUG, "No limit is set on " - "volume %s", this_vol->name); - /* Dafault ret is 0 */ - goto err; - } + lookup_cbk (this, frame, &rsp); - ret = synctask_new (this->ctx->env, - qd_trigger_periodically, - qd_trigger_periodically_cbk, - NULL, (void *) this_vol); - if (-1 == ret) { - gf_log (this->name, GF_LOG_ERROR, "Synctask creation " - "failed for %s", this_vol->name); - goto err; - } -err: - return ret; + inode_unref (loc.inode); + return 0; } int @@ -749,12 +150,6 @@ qd_init (xlator_t *this) { int32_t ret = -1; quota_priv_t *priv = NULL; - int i = 0; - char *option_str = NULL; - xlator_list_t *subvol = NULL; - char *limits = NULL; - int subvol_cnt = 0; - qd_vols_conf_t *this_vol = NULL; if (NULL == this->children) { gf_log (this->name, GF_LOG_ERROR, @@ -765,183 +160,27 @@ qd_init (xlator_t *this) } QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err); - LOCK_INIT (&priv->lock); this->private = priv; - for (subvol_cnt = 0, subvol = this->children; - subvol; - subvol_cnt++, subvol = subvol->next); - - priv->qd_vols_conf = GF_CALLOC (sizeof (qd_vols_conf_t *), subvol_cnt, - gf_quota_mt_qd_vols_conf_t); - if (!priv->qd_vols_conf) { - gf_log (this->name, GF_LOG_ERROR, "Failed to allocate memory"); - goto err; - } - - for (i = 0, subvol = this->children; - subvol; - subvol = subvol->next, i++) { - - QUOTA_ALLOC_OR_GOTO (priv->qd_vols_conf[i], - qd_vols_conf_t, err); - - this_vol = priv->qd_vols_conf[i]; - - LOCK_INIT (&this_vol->lock); - INIT_LIST_HEAD (&this_vol->limit_head); - - this_vol->name = subvol->xlator->name; - - ret = gf_asprintf (&option_str, "%s.default-soft-limit", - this_vol->name); - if (0 > ret) { - gf_log (this->name, GF_LOG_ERROR, - "gf_asprintf failed"); - goto err; - } - GF_OPTION_INIT (option_str, this_vol->default_soft_lim, percent, - err); - GF_FREE (option_str); - - ret = gf_asprintf (&option_str, "%s.alert-time", - this_vol->name); - if (0 > ret) { - gf_log (this->name, GF_LOG_ERROR, - "gf_asprintf failed"); - goto err; - } - GF_OPTION_INIT (option_str, this_vol->log_timeout, time, err); - GF_FREE (option_str); - - ret = gf_asprintf (&option_str, "%s.limit-set", this_vol->name); - if (0 > ret) { - gf_log (this->name, GF_LOG_ERROR, - "gf_asprintf failed"); - goto err; - } - ret = dict_get_str (this->options, option_str, &limits); - if (ret < 0) { - gf_log (this->name, GF_LOG_ERROR, "dict get failed or " - "no limits set"); - continue; - } - - ret = qd_parse_limits (priv, this, limits, NULL, this_vol); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "Couldn't parse limits for %s", this_vol->name); - goto err; - } - GF_FREE (option_str); - - ret = gf_asprintf (&option_str, "%s.soft-timeout", - this_vol->name); - if (0 > ret) { - gf_log (this->name, GF_LOG_ERROR, - "gf_asprintf failed"); - goto err; - } - GF_OPTION_INIT (option_str, this_vol->soft_timeout, - time, err); - GF_FREE (option_str); - - ret = gf_asprintf (&option_str, "%s.hard-timeout", - this_vol->name); - if (0 > ret) { - gf_log (this->name, GF_LOG_ERROR, - "gf_asprintf failed"); - goto err; - } - GF_OPTION_INIT (option_str, this_vol->hard_timeout, - time, err); - GF_FREE (option_str); - } - - this->local_pool = mem_pool_new (quota_local_t, 64); - if (!this->local_pool) { - ret = -1; - gf_log (this->name, GF_LOG_ERROR, - "failed to create local_t's memory pool"); + ret = quotad_aggregator_init (this); + if (ret < 0) goto err; - } ret = 0; err: - /* Free all allocated variables */ if (ret) { - /* can reach here from GF_OPTION_INIT, so cleaning opt_str */ - GF_FREE (option_str); - - for (i = 0; i < subvol_cnt; i++) - GF_FREE (priv->qd_vols_conf[i]); - GF_FREE (priv->qd_vols_conf); - GF_FREE (priv); } return ret; } -int -qd_notify (xlator_t *this, int event, void *data, ...) -{ - xlator_list_t *subvol = NULL; - xlator_t *subvol_rec = NULL; - quota_priv_t *priv = NULL; - int i = 0; - int ret = 0; - - subvol_rec = data; - priv = this->private; - - for (i=0, subvol = this->children; subvol; i++, subvol = subvol->next) { - if (! strcmp (priv->qd_vols_conf[i]->name, subvol_rec->name)) - break; - } - if (!subvol) { - default_notify (this, event, data); - goto out; - } - - switch (event) { - case GF_EVENT_CHILD_UP: - { - /* handle spurious CHILD_UP and DOWN events */ - if (!priv->qd_vols_conf[i]->threads_status) { - ret = qd_start_threads (this, i); - if (-1 == ret) { - gf_log (this->name, GF_LOG_ERROR, "Couldn't " - "start the threads for volumes"); - goto out; - } - - priv->qd_vols_conf[i]->threads_status = _gf_true; - } - break; - } - case GF_EVENT_CHILD_DOWN: - { - gf_log (this->name, GF_LOG_ERROR, "vol %s is down.", - priv->qd_vols_conf [i]->name); - break; - } - default: - default_notify (this, event, data); - }/* end switch */ - - - -out: - return ret; -} - class_methods_t class_methods = { .init = qd_init, .fini = qd_fini, .reconfigure = qd_reconfigure, - .notify = qd_notify, +// .notify = qd_notify, }; struct xlator_fops fops = { @@ -951,36 +190,17 @@ struct xlator_cbks cbks = { }; struct volume_options options[] = { - {.key = {"*.limit-set"}}, - {.key = {"*.soft-timeout"}, - .type = GF_OPTION_TYPE_TIME, - .min = 1, - .max = LONG_MAX, - .default_value = "10", - .description = "" - }, - {.key = {"*.hard-timeout"}, - .type = GF_OPTION_TYPE_TIME, - .min = 0, - .max = LONG_MAX, - .default_value = "2", - .description = "" - }, - {.key = {"*.alert-time"}, - .type = GF_OPTION_TYPE_TIME, - .min = 0, - .max = LONG_MAX, - /* default weekly (7 * 24 * 60 *60) */ - .default_value = "604800", - .description = "" + { .key = {"transport-type"}, + .value = {"rpc", "rpc-over-rdma", "tcp", "socket", "ib-verbs", + "unix", "ib-sdp", "tcp/server", "ib-verbs/server", "rdma", + "rdma*([ \t]),*([ \t])socket", + "rdma*([ \t]),*([ \t])tcp", + "tcp*([ \t]),*([ \t])rdma", + "socket*([ \t]),*([ \t])rdma"}, + .type = GF_OPTION_TYPE_STR }, - {.key = {"*.default-soft-limit"}, - .type = GF_OPTION_TYPE_PERCENT, - .min = 0, - .max = 100, - .default_value = "90%", - .description = "Takes this if individual paths are not configured " - "with soft limits." + { .key = {"transport.*"}, + .type = GF_OPTION_TYPE_ANY, }, {.key = {NULL}} }; -- cgit