summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDan Lambright <dlambrig@redhat.com>2016-05-25 13:54:13 -0400
committerDan Lambright <dlambrig@redhat.com>2016-07-27 06:03:48 -0700
commit64ae3907818ad3cdaed1a1454cb44f684c9fcffd (patch)
treebd6a746f5a48c0463e981b2fd225bbc1956a5ddb
parentdbc8dac5c357a28aeba17894f8b02faa2af2a792 (diff)
cluster/tier: fix statfs for dht/tiered volumes
Return the correct size of the tiered volume in statfs. It should be the size of the cold tier, not the sum of the hot and cold tier, because the hot tier is a cache and not an extension of the volume's capacity. The number of free blocks, etc is the cold tier's capacity subtracted by the sum of utilization on the hot and cold tiers. Note if both tiers are part of the same file system this must be accounted for as well. The patch also fixes a pre-existing bug in the DHT/tier translators. If statfs was taken on a file, the code only calculated free space on the cached subvolume, not all subvolumes in the replica group. With the fix, this is corrected, except in the case where quota is used with the deem-statfs option set to "on". Change-Id: I2b8bcb4511edf83f12130960aad0a609fcf8f513 BUG: 1339689 Signed-off-by: Dan Lambright <dlambrig@redhat.com> Reviewed-on: http://review.gluster.org/14536 Smoke: Gluster Build System <jenkins@build.gluster.org> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> Reviewed-by: Raghavendra G <rgowdapp@redhat.com> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
-rw-r--r--xlators/cluster/dht/src/dht-common.c47
-rw-r--r--xlators/cluster/dht/src/dht-common.h18
-rw-r--r--xlators/cluster/dht/src/dht-rebalance.c1
-rw-r--r--xlators/cluster/dht/src/tier-common.c213
-rw-r--r--xlators/cluster/dht/src/tier-common.h6
-rw-r--r--xlators/cluster/dht/src/tier.c2
6 files changed, 265 insertions, 22 deletions
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index c667266fed8..f3499ed6d0b 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -4387,7 +4387,7 @@ unlock:
/*
* dht_normalize_stats -
*/
-static void
+void
dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
unsigned long frsize)
{
@@ -4535,6 +4535,10 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
dht_conf_t *conf = NULL;
int op_errno = -1;
int i = -1;
+ inode_t *inode = NULL;
+ inode_table_t *itable = NULL;
+ uuid_t root_gfid = {0, };
+ loc_t newloc = {0, };
VALIDATE_OR_GOTO (frame, err);
VALIDATE_OR_GOTO (this, err);
@@ -4549,31 +4553,34 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
goto err;
}
- if (!loc->inode || IA_ISDIR (loc->inode->ia_type)) {
- local->call_cnt = conf->subvolume_cnt;
+ if (loc->inode && !IA_ISDIR (loc->inode->ia_type)) {
+ itable = loc->inode->table;
+ if (!itable) {
+ op_errno = EINVAL;
+ goto err;
+ }
- for (i = 0; i < conf->subvolume_cnt; i++) {
- STACK_WIND (frame, dht_statfs_cbk,
- conf->subvolumes[i],
- conf->subvolumes[i]->fops->statfs, loc,
- xdata);
+ loc = &local->loc2;
+ root_gfid[15] = 1;
+
+ inode = inode_find (itable, root_gfid);
+ if (!inode) {
+ op_errno = EINVAL;
+ goto err;
}
- return 0;
- }
- subvol = dht_subvol_get_cached (this, loc->inode);
- if (!subvol) {
- gf_msg_debug (this->name, 0,
- "no cached subvolume for path=%s", loc->path);
- op_errno = EINVAL;
- goto err;
+ dht_build_root_loc (inode, &newloc);
+ loc = &newloc;
}
- local->call_cnt = 1;
-
- STACK_WIND (frame, dht_statfs_cbk,
- subvol, subvol->fops->statfs, loc, xdata);
+ local->call_cnt = conf->subvolume_cnt;
+ for (i = 0; i < conf->subvolume_cnt; i++) {
+ STACK_WIND (frame, dht_statfs_cbk,
+ conf->subvolumes[i],
+ conf->subvolumes[i]->fops->statfs, loc,
+ xdata);
+ }
return 0;
err:
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 9a71c46c8e4..080e98594ec 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -177,6 +177,19 @@ typedef
gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame, dht_layout_t **inmem,
dht_layout_t **ondisk);
+typedef struct {
+ uint64_t blocks_used;
+ uint64_t pblocks_used;
+ uint64_t files_used;
+ uint64_t pfiles_used;
+ uint64_t unhashed_blocks_used;
+ uint64_t unhashed_pblocks_used;
+ uint64_t unhashed_files_used;
+ uint64_t unhashed_pfiles_used;
+ uint64_t unhashed_fsid;
+ uint64_t hashed_fsid;
+} tier_statvfs_t;
+
struct dht_local {
int call_cnt;
loc_t loc;
@@ -193,6 +206,7 @@ struct dht_local {
struct iatt preparent;
struct iatt postparent;
struct statvfs statvfs;
+ tier_statvfs_t tier_statvfs;
fd_t *fd;
inode_t *inode;
dict_t *params;
@@ -1229,4 +1243,8 @@ dht_get_lock_subvolume (xlator_t *this, struct gf_flock *lock,
int
dht_lk_inode_unref (call_frame_t *frame, int32_t op_ret);
+void
+dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
+ unsigned long frsize);
+
#endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 4c83ed478c0..83805fe86a6 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -1917,7 +1917,6 @@ dht_build_root_loc (inode_t *inode, loc_t *loc)
loc->gfid[15] = 1;
}
-
/* return values: 1 -> error, bug ignore and continue
0 -> proceed
-1 -> error, handle it */
diff --git a/xlators/cluster/dht/src/tier-common.c b/xlators/cluster/dht/src/tier-common.c
index 20d3f24d3bf..5b8634718ca 100644
--- a/xlators/cluster/dht/src/tier-common.c
+++ b/xlators/cluster/dht/src/tier-common.c
@@ -1082,3 +1082,216 @@ tier_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
tier_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
return 0;
}
+
+int
+tier_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+ int op_ret, int op_errno, struct statvfs *statvfs,
+ dict_t *xdata)
+{
+ gf_boolean_t event = _gf_false;
+ qdstatfs_action_t action = qdstatfs_action_OFF;
+ dht_local_t *local = NULL;
+ int this_call_cnt = 0;
+ int bsize = 0;
+ int frsize = 0;
+ GF_UNUSED int ret = 0;
+ unsigned long new_usage = 0;
+ unsigned long cur_usage = 0;
+ call_frame_t *prev = NULL;
+ dht_conf_t *conf = NULL;
+ tier_statvfs_t *tier_stat = NULL;
+
+ prev = cookie;
+ local = frame->local;
+ GF_ASSERT (local);
+
+ conf = this->private;
+
+ if (xdata)
+ ret = dict_get_int8 (xdata, "quota-deem-statfs",
+ (int8_t *)&event);
+
+ tier_stat = &local->tier_statvfs;
+
+ LOCK (&frame->lock);
+ {
+ if (op_ret == -1) {
+ local->op_errno = op_errno;
+ goto unlock;
+ }
+ if (!statvfs) {
+ op_errno = EINVAL;
+ local->op_ret = -1;
+ goto unlock;
+ }
+ local->op_ret = 0;
+
+ switch (local->quota_deem_statfs) {
+ case _gf_true:
+ if (event == _gf_true)
+ action = qdstatfs_action_COMPARE;
+ else
+ action = qdstatfs_action_NEGLECT;
+ break;
+
+ case _gf_false:
+ if (event == _gf_true) {
+ action = qdstatfs_action_REPLACE;
+ local->quota_deem_statfs = _gf_true;
+ }
+ break;
+
+ default:
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ DHT_MSG_INVALID_VALUE,
+ "Encountered third "
+ "value for boolean variable %d",
+ local->quota_deem_statfs);
+ break;
+ }
+
+ if (local->quota_deem_statfs) {
+ switch (action) {
+ case qdstatfs_action_NEGLECT:
+ goto unlock;
+
+ case qdstatfs_action_REPLACE:
+ local->statvfs = *statvfs;
+ goto unlock;
+
+ case qdstatfs_action_COMPARE:
+ new_usage = statvfs->f_blocks -
+ statvfs->f_bfree;
+ cur_usage = local->statvfs.f_blocks -
+ local->statvfs.f_bfree;
+
+ /* Take the max of the usage from subvols */
+ if (new_usage >= cur_usage)
+ local->statvfs = *statvfs;
+ goto unlock;
+
+ default:
+ break;
+ }
+ }
+
+ if (local->statvfs.f_bsize != 0) {
+ bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
+ frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
+ dht_normalize_stats(&local->statvfs, bsize, frsize);
+ dht_normalize_stats(statvfs, bsize, frsize);
+ } else {
+ local->statvfs.f_bsize = statvfs->f_bsize;
+ local->statvfs.f_frsize = statvfs->f_frsize;
+ }
+
+ if (prev->this == TIER_HASHED_SUBVOL) {
+ local->statvfs.f_blocks = statvfs->f_blocks;
+ local->statvfs.f_files = statvfs->f_files;
+ local->statvfs.f_fsid = statvfs->f_fsid;
+ local->statvfs.f_flag = statvfs->f_flag;
+ local->statvfs.f_namemax = statvfs->f_namemax;
+ tier_stat->blocks_used = (statvfs->f_blocks - statvfs->f_bfree);
+ tier_stat->pblocks_used = (statvfs->f_blocks - statvfs->f_bavail);
+ tier_stat->files_used = (statvfs->f_files - statvfs->f_ffree);
+ tier_stat->pfiles_used = (statvfs->f_files - statvfs->f_favail);
+ tier_stat->hashed_fsid = statvfs->f_fsid;
+ } else {
+ tier_stat->unhashed_fsid = statvfs->f_fsid;
+ tier_stat->unhashed_blocks_used = (statvfs->f_blocks - statvfs->f_bfree);
+ tier_stat->unhashed_pblocks_used = (statvfs->f_blocks - statvfs->f_bavail);
+ tier_stat->unhashed_files_used = (statvfs->f_files - statvfs->f_ffree);
+ tier_stat->unhashed_pfiles_used = (statvfs->f_files - statvfs->f_favail);
+ }
+
+ }
+unlock:
+ UNLOCK (&frame->lock);
+
+ this_call_cnt = dht_frame_return (frame);
+ if (is_last_call (this_call_cnt)) {
+ if (tier_stat->unhashed_fsid != tier_stat->hashed_fsid) {
+ tier_stat->blocks_used += tier_stat->unhashed_blocks_used;
+ tier_stat->pblocks_used += tier_stat->unhashed_pblocks_used;
+ tier_stat->files_used += tier_stat->unhashed_files_used;
+ tier_stat->pfiles_used += tier_stat->unhashed_pfiles_used;
+ }
+ local->statvfs.f_bfree = local->statvfs.f_blocks -
+ tier_stat->blocks_used;
+ local->statvfs.f_bavail = local->statvfs.f_blocks -
+ tier_stat->pblocks_used;
+ local->statvfs.f_ffree = local->statvfs.f_files -
+ tier_stat->files_used;
+ local->statvfs.f_favail = local->statvfs.f_files -
+ tier_stat->pfiles_used;
+ DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
+ &local->statvfs, xdata);
+ }
+
+ return 0;
+}
+
+
+int
+tier_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+ xlator_t *subvol = NULL;
+ dht_local_t *local = NULL;
+ dht_conf_t *conf = NULL;
+ int op_errno = -1;
+ int i = -1;
+ inode_t *inode = NULL;
+ inode_table_t *itable = NULL;
+ uuid_t root_gfid = {0, };
+ loc_t newloc = {0, };
+
+ VALIDATE_OR_GOTO (frame, err);
+ VALIDATE_OR_GOTO (this, err);
+ VALIDATE_OR_GOTO (loc, err);
+ VALIDATE_OR_GOTO (this->private, err);
+
+ conf = this->private;
+
+ local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
+ if (!local) {
+ op_errno = ENOMEM;
+ goto err;
+ }
+
+ if (loc->inode && !IA_ISDIR (loc->inode->ia_type)) {
+ itable = loc->inode->table;
+ if (!itable) {
+ op_errno = EINVAL;
+ goto err;
+ }
+
+ loc = &local->loc2;
+ root_gfid[15] = 1;
+
+ inode = inode_find (itable, root_gfid);
+ if (!inode) {
+ op_errno = EINVAL;
+ goto err;
+ }
+
+ dht_build_root_loc (inode, &newloc);
+ loc = &newloc;
+ }
+
+ local->call_cnt = conf->subvolume_cnt;
+
+ for (i = 0; i < conf->subvolume_cnt; i++) {
+ STACK_WIND (frame, tier_statfs_cbk,
+ conf->subvolumes[i],
+ conf->subvolumes[i]->fops->statfs, loc,
+ xdata);
+ }
+
+ return 0;
+
+err:
+ op_errno = (op_errno == -1) ? errno : op_errno;
+ DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+
+ return 0;
+}
diff --git a/xlators/cluster/dht/src/tier-common.h b/xlators/cluster/dht/src/tier-common.h
index 0ef96aca032..26e01d7f141 100644
--- a/xlators/cluster/dht/src/tier-common.h
+++ b/xlators/cluster/dht/src/tier-common.h
@@ -58,5 +58,11 @@ tier_readdir (call_frame_t *frame,
int
tier_link (call_frame_t *frame, xlator_t *this,
loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+
+
+int
+tier_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+
#endif
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
index 356af021563..9c8f15bbce2 100644
--- a/xlators/cluster/dht/src/tier.c
+++ b/xlators/cluster/dht/src/tier.c
@@ -2465,7 +2465,7 @@ struct xlator_fops fops = {
.mknod = dht_mknod,
.open = dht_open,
- .statfs = dht_statfs,
+ .statfs = tier_statfs,
.opendir = dht_opendir,
.readdir = tier_readdir,
.readdirp = tier_readdirp,