rebalance: ``check_free_space`` should ignore quota_statfs

quota_statfs() returns aggregated details of space usage
of bricks this causes distribute to be confused during
``rebalance``, where ``statfs()`` values are used to
schedule file migration.

We can make sure the values of ``statfs`` are from
individual bricks by selectively instructing
``quota_statfs()`` to return non aggregated values.

Change-Id: I1397faeee66a1b9c26709cfda693286d227a4170
BUG: 1158262
Signed-off-by: Harshavardhana <harsha@harshavardhana.net>
Reviewed-on: http://review.gluster.org/8996
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: N Balachandran <nbalacha@redhat.com>
Reviewed-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-by: Shyamsundar Ranganathan <srangana@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

4 files changed, 46 insertions, 14 deletions

diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index a5ab8e43961..66a3df2b2d1 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -2172,7 +2172,7 @@ retry:
 	if (ret)
 		goto out;
 
-	ret = syncop_statfs (subvol, &loc, buf);
+	ret = syncop_statfs (subvol, &loc, NULL, buf, NULL);
         DECODE_SYNCOP_ERR (ret);
 
 	ESTALE_RETRY (ret, errno, reval, &loc, retry);
diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index 3e176cffc17..e830d830229 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -1438,6 +1438,8 @@ syncop_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (op_ret == 0) {
                 args->statvfs_buf  = *buf;
+                if (xdata)
+                        args->xdata  = dict_ref (xdata);
         }
 
         __wake (args);
@@ -1447,16 +1449,21 @@ syncop_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 
 int
-syncop_statfs (xlator_t *subvol, loc_t *loc, struct statvfs *buf)
+syncop_statfs (xlator_t *subvol, loc_t *loc, dict_t *xdata_req,
+               struct statvfs *buf, dict_t **xdata_rsp)
 
 {
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_statfs_cbk, subvol->fops->statfs,
-                loc, NULL);
+                loc, xdata_req);
 
         if (buf)
                 *buf = args.statvfs_buf;
+        if (xdata_rsp)
+                *xdata_rsp = args.xdata;
+        else if (args.xdata)
+                dict_unref (args.xdata);
 
         if (args.op_ret < 0)
                 return -args.op_errno;
diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h
index 31438b17883..249abb3d331 100644
--- a/libglusterfs/src/syncop.h
+++ b/libglusterfs/src/syncop.h
@@ -361,7 +361,9 @@ int syncop_fsetattr (xlator_t *subvol, fd_t *fd, struct iatt *iatt, int valid,
                     /* out */
                     struct iatt *preop, struct iatt *postop);
 
-int syncop_statfs (xlator_t *subvol, loc_t *loc, struct statvfs *buf);
+int syncop_statfs (xlator_t *subvol, loc_t *loc, dict_t *xattr_req,
+                   /* out */
+                   struct statvfs *buf, dict_t **xattr_rsp);
 
 int syncop_setxattr (xlator_t *subvol, loc_t *loc, dict_t *dict, int32_t flags);
 int syncop_fsetxattr (xlator_t *subvol, fd_t *fd, dict_t *dict, int32_t flags);
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 3c976b113fa..85499737e30 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -491,13 +491,31 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         struct statvfs  dst_statfs = {0,};
         int             ret        = -1;
         xlator_t       *this       = NULL;
+        dict_t         *xdata      = NULL;
 
         uint64_t        src_statfs_blocks = 1;
         uint64_t        dst_statfs_blocks = 1;
 
         this = THIS;
 
-        ret = syncop_statfs (from, loc, &src_statfs);
+        xdata = dict_new ();
+        if (!xdata) {
+                errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to allocate dictionary");
+                goto out;
+        }
+
+        ret = dict_set_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set "
+                        GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_statfs (from, loc, xdata, &src_statfs, NULL);
         if (ret) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
                         DHT_MSG_MIGRATE_FILE_FAILED,
@@ -507,7 +525,7 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
                 goto out;
         }
 
-        ret = syncop_statfs (to, loc, &dst_statfs);
+        ret = syncop_statfs (to, loc, xdata, &dst_statfs, NULL);
         if (ret) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
                         DHT_MSG_MIGRATE_FILE_FAILED,
@@ -541,10 +559,13 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
 
                         gf_msg (this->name, GF_LOG_WARNING, 0,
                                 DHT_MSG_MIGRATE_FILE_FAILED,
-                                "data movement attempted from node (%s) with"
-                                " higher disk space to a node (%s) with "
-                                "lesser disk space (%s)", from->name,
-                                to->name, loc->path);
+                                "data movement attempted from node "
+                                "(%s:%"PRIu64") with higher disk space "
+                                "to a node (%s:%"PRIu64") with lesser "
+                                "disk space, file { blocks:%"PRIu64", "
+                                "name:(%s) }", from->name, src_statfs_blocks,
+                                to->name, dst_statfs_blocks,
+                                stbuf->ia_blocks, loc->path);
 
                         /* this is not a 'failure', but we don't want to
                            consider this as 'success' too :-/ */
@@ -557,15 +578,17 @@ check_avail_space:
               GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
                         DHT_MSG_MIGRATE_FILE_FAILED,
-                        "data movement attempted from node (%s) with "
-                        "to node (%s) which does not have required free space"
-                        " for %s", from->name, to->name, loc->path);
+                        "data movement attempted from node (%s) to node (%s) "
+                        "which does not have required free space for (%s)",
+                        from->name, to->name, loc->path);
                 ret = -1;
                 goto out;
         }
 
         ret = 0;
 out:
+        if (xdata)
+                dict_unref (xdata);
         return ret;
 }
 
@@ -1510,7 +1533,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if (ret) {
                                 gf_msg (this->name, GF_LOG_ERROR, 0,
                                         DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "Migrate file failed:%s lookup failed", 
+                                        "Migrate file failed:%s lookup failed",
                                         entry_loc.path);
                                 ret = -1;
                                 continue;