cluster/tier: fix statfs for dht/tiered volumes

Return the correct size of the tiered volume in statfs. It should
be the size of the cold tier, not the sum of the hot and cold tier,
because the hot tier is a cache and not an extension of the volume's
capacity. The number of free blocks, etc is the cold tier's capacity
subtracted by the sum of utilization on the hot and cold tiers. Note
if both tiers are part of the same file system this must be accounted
for as well.

The patch also fixes a pre-existing bug in the DHT/tier
translators. If statfs was taken on a file, the code only calculated
free space on the cached subvolume, not all subvolumes in the replica
group. With the fix, this is corrected, except in the case
where quota is used with the deem-statfs option set to "on".

Change-Id: I2b8bcb4511edf83f12130960aad0a609fcf8f513
BUG: 1339689
Signed-off-by: Dan Lambright <dlambrig@redhat.com>
Reviewed-on: http://review.gluster.org/14536
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

6 files changed, 265 insertions, 22 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index c667266fed8..f3499ed6d0b 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -4387,7 +4387,7 @@ unlock:
 /*
  * dht_normalize_stats -
  */
-static void
+void
 dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
                      unsigned long frsize)
 {
@@ -4535,6 +4535,10 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
         dht_conf_t   *conf = NULL;
         int           op_errno = -1;
         int           i = -1;
+        inode_t          *inode         = NULL;
+        inode_table_t    *itable        = NULL;
+        uuid_t            root_gfid     = {0, };
+        loc_t         newloc = {0, };
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -4549,31 +4553,34 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
                 goto err;
         }
 
-        if (!loc->inode || IA_ISDIR (loc->inode->ia_type)) {
-                local->call_cnt = conf->subvolume_cnt;
+        if (loc->inode && !IA_ISDIR (loc->inode->ia_type)) {
+                itable = loc->inode->table;
+                if (!itable) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        STACK_WIND (frame, dht_statfs_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->statfs, loc,
-                                    xdata);
+                loc = &local->loc2;
+                root_gfid[15] = 1;
+
+                inode = inode_find (itable, root_gfid);
+                if (!inode) {
+                        op_errno = EINVAL;
+                        goto err;
                 }
-                return 0;
-        }
 
-        subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+                dht_build_root_loc (inode, &newloc);
+                loc = &newloc;
         }
 
-        local->call_cnt = 1;
-
-        STACK_WIND (frame, dht_statfs_cbk,
-                    subvol, subvol->fops->statfs, loc, xdata);
+        local->call_cnt = conf->subvolume_cnt;
 
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_statfs_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->statfs, loc,
+                            xdata);
+        }
         return 0;
 
 err:
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 9a71c46c8e4..080e98594ec 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -177,6 +177,19 @@ typedef
 gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame, dht_layout_t **inmem,
                                 dht_layout_t **ondisk);
 
+typedef struct {
+        uint64_t                 blocks_used;
+        uint64_t                 pblocks_used;
+        uint64_t                 files_used;
+        uint64_t                 pfiles_used;
+        uint64_t                 unhashed_blocks_used;
+        uint64_t                 unhashed_pblocks_used;
+        uint64_t                 unhashed_files_used;
+        uint64_t                 unhashed_pfiles_used;
+        uint64_t                 unhashed_fsid;
+        uint64_t                 hashed_fsid;
+} tier_statvfs_t;
+
 struct dht_local {
         int                      call_cnt;
         loc_t                    loc;
@@ -193,6 +206,7 @@ struct dht_local {
         struct iatt              preparent;
         struct iatt              postparent;
         struct statvfs           statvfs;
+        tier_statvfs_t           tier_statvfs;
         fd_t                    *fd;
         inode_t                 *inode;
         dict_t                  *params;
@@ -1229,4 +1243,8 @@ dht_get_lock_subvolume (xlator_t *this, struct gf_flock *lock,
 int
 dht_lk_inode_unref (call_frame_t *frame, int32_t op_ret);
 
+void
+dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
+                     unsigned long frsize);
+
 #endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 4c83ed478c0..83805fe86a6 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -1917,7 +1917,6 @@ dht_build_root_loc (inode_t *inode, loc_t *loc)
         loc->gfid[15] = 1;
 }
 
-
 /* return values: 1 -> error, bug ignore and continue
                   0 -> proceed
                  -1 -> error, handle it */
diff --git a/xlators/cluster/dht/src/tier-common.c b/xlators/cluster/dht/src/tier-common.c
index 20d3f24d3bf..5b8634718ca 100644
--- a/xlators/cluster/dht/src/tier-common.c
+++ b/xlators/cluster/dht/src/tier-common.c
@@ -1082,3 +1082,216 @@ tier_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         tier_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
         return 0;
 }
+
+int
+tier_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct statvfs *statvfs,
+                dict_t *xdata)
+{
+        gf_boolean_t            event              = _gf_false;
+        qdstatfs_action_t       action             = qdstatfs_action_OFF;
+        dht_local_t             *local             = NULL;
+        int                     this_call_cnt      = 0;
+        int                     bsize              = 0;
+        int                     frsize             = 0;
+        GF_UNUSED int           ret                = 0;
+        unsigned long           new_usage          = 0;
+        unsigned long           cur_usage          = 0;
+        call_frame_t            *prev              = NULL;
+        dht_conf_t              *conf              = NULL;
+        tier_statvfs_t          *tier_stat         = NULL;
+
+        prev = cookie;
+        local = frame->local;
+        GF_ASSERT (local);
+
+        conf = this->private;
+
+        if (xdata)
+                ret = dict_get_int8 (xdata, "quota-deem-statfs",
+                                     (int8_t *)&event);
+
+        tier_stat = &local->tier_statvfs;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        goto unlock;
+                }
+                if (!statvfs) {
+                        op_errno = EINVAL;
+                        local->op_ret = -1;
+                        goto unlock;
+                }
+                local->op_ret = 0;
+
+                switch (local->quota_deem_statfs) {
+                case _gf_true:
+                        if (event == _gf_true)
+                                action = qdstatfs_action_COMPARE;
+                        else
+                                action = qdstatfs_action_NEGLECT;
+                        break;
+
+                case _gf_false:
+                        if (event == _gf_true) {
+                                action = qdstatfs_action_REPLACE;
+                                local->quota_deem_statfs = _gf_true;
+                        }
+                        break;
+
+                default:
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                DHT_MSG_INVALID_VALUE,
+                                "Encountered third "
+                                "value for boolean variable %d",
+                                local->quota_deem_statfs);
+                        break;
+                }
+
+                if (local->quota_deem_statfs) {
+                        switch (action) {
+                        case qdstatfs_action_NEGLECT:
+                                goto unlock;
+
+                        case qdstatfs_action_REPLACE:
+                                local->statvfs = *statvfs;
+                                goto unlock;
+
+                        case qdstatfs_action_COMPARE:
+                                new_usage = statvfs->f_blocks -
+                                             statvfs->f_bfree;
+                                cur_usage = local->statvfs.f_blocks -
+                                             local->statvfs.f_bfree;
+
+                                /* Take the max of the usage from subvols */
+                                if (new_usage >= cur_usage)
+                                        local->statvfs = *statvfs;
+                                goto unlock;
+
+                        default:
+                                break;
+                        }
+                }
+
+                if (local->statvfs.f_bsize != 0) {
+                        bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
+                        frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
+                        dht_normalize_stats(&local->statvfs, bsize, frsize);
+                        dht_normalize_stats(statvfs, bsize, frsize);
+                } else {
+                        local->statvfs.f_bsize    = statvfs->f_bsize;
+                        local->statvfs.f_frsize   = statvfs->f_frsize;
+                }
+
+                if (prev->this == TIER_HASHED_SUBVOL) {
+                        local->statvfs.f_blocks   = statvfs->f_blocks;
+                        local->statvfs.f_files    = statvfs->f_files;
+                        local->statvfs.f_fsid     = statvfs->f_fsid;
+                        local->statvfs.f_flag     = statvfs->f_flag;
+                        local->statvfs.f_namemax  = statvfs->f_namemax;
+                        tier_stat->blocks_used    = (statvfs->f_blocks - statvfs->f_bfree);
+                        tier_stat->pblocks_used   = (statvfs->f_blocks - statvfs->f_bavail);
+                        tier_stat->files_used     = (statvfs->f_files - statvfs->f_ffree);
+                        tier_stat->pfiles_used    = (statvfs->f_files - statvfs->f_favail);
+                        tier_stat->hashed_fsid    = statvfs->f_fsid;
+                } else {
+                        tier_stat->unhashed_fsid      = statvfs->f_fsid;
+                        tier_stat->unhashed_blocks_used    = (statvfs->f_blocks - statvfs->f_bfree);
+                        tier_stat->unhashed_pblocks_used   = (statvfs->f_blocks - statvfs->f_bavail);
+                        tier_stat->unhashed_files_used     = (statvfs->f_files - statvfs->f_ffree);
+                        tier_stat->unhashed_pfiles_used    = (statvfs->f_files - statvfs->f_favail);
+                }
+
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                if (tier_stat->unhashed_fsid != tier_stat->hashed_fsid) {
+                        tier_stat->blocks_used    += tier_stat->unhashed_blocks_used;
+                        tier_stat->pblocks_used   += tier_stat->unhashed_pblocks_used;
+                        tier_stat->files_used     += tier_stat->unhashed_files_used;
+                        tier_stat->pfiles_used    += tier_stat->unhashed_pfiles_used;
+                }
+                local->statvfs.f_bfree = local->statvfs.f_blocks -
+                        tier_stat->blocks_used;
+                local->statvfs.f_bavail = local->statvfs.f_blocks -
+                        tier_stat->pblocks_used;
+                local->statvfs.f_ffree = local->statvfs.f_files -
+                        tier_stat->files_used;
+                local->statvfs.f_favail = local->statvfs.f_files -
+                        tier_stat->pfiles_used;
+                DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
+                                  &local->statvfs, xdata);
+        }
+
+        return 0;
+}
+
+
+int
+tier_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        xlator_t         *subvol = NULL;
+        dht_local_t      *local  = NULL;
+        dht_conf_t       *conf = NULL;
+        int               op_errno = -1;
+        int               i = -1;
+        inode_t          *inode         = NULL;
+        inode_table_t    *itable        = NULL;
+        uuid_t            root_gfid     = {0, };
+        loc_t             newloc        = {0, };
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
+
+        local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        if (loc->inode && !IA_ISDIR (loc->inode->ia_type)) {
+                itable = loc->inode->table;
+                if (!itable) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                loc = &local->loc2;
+                root_gfid[15] = 1;
+
+                inode = inode_find (itable, root_gfid);
+                if (!inode) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                dht_build_root_loc (inode, &newloc);
+                loc = &newloc;
+        }
+
+        local->call_cnt = conf->subvolume_cnt;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, tier_statfs_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->statfs, loc,
+                            xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/tier-common.h b/xlators/cluster/dht/src/tier-common.h
index 0ef96aca032..26e01d7f141 100644
--- a/xlators/cluster/dht/src/tier-common.h
+++ b/xlators/cluster/dht/src/tier-common.h
@@ -58,5 +58,11 @@ tier_readdir (call_frame_t *frame,
 int
 tier_link (call_frame_t *frame, xlator_t *this,
           loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+
+
+int
+tier_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+
 #endif
 
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
index 356af021563..9c8f15bbce2 100644
--- a/xlators/cluster/dht/src/tier.c
+++ b/xlators/cluster/dht/src/tier.c
@@ -2465,7 +2465,7 @@ struct xlator_fops fops = {
         .mknod       = dht_mknod,
 
         .open        = dht_open,
-        .statfs      = dht_statfs,
+        .statfs      = tier_statfs,
         .opendir     = dht_opendir,
         .readdir     = tier_readdir,
         .readdirp    = tier_readdirp,