cluster/dht: Treat migration failures due to space constraints as skippedv3.4.1qa3

Currently rebalance/remove-brick op's display migration failed count even
for files which failed due to space issues (not enough space for file, or
migration leading to cluster imbalance)

These will now be counted as skipped, and rebalance/remove-brick status
will display the additional counter

BUG: 989846
Change-Id: I4efa7ce69dd43680ff47181afed0c561954c5080
Signed-off-by: shishir gowda <sgowda@redhat.com>
Reviewed-on: http://review.gluster.org/5977
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

6 files changed, 86 insertions, 17 deletions

diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 89d8a18aafe..81692ffa532 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -1239,6 +1239,7 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                     key[256] = {0,};
         int32_t                  i = 1;
         uint64_t                 failures = 0;
+        uint64_t                 skipped = 0;
         double                   elapsed = 0;
         char                    *size_str = NULL;
         char                    *task_id_str = NULL;
@@ -1354,11 +1355,12 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-        cli_out ("%40s %16s %13s %13s %13s %14s %s", "Node", "Rebalanced-files",
-                 "size", "scanned", "failures", "status", "run time in secs");
-        cli_out ("%40s %16s %13s %13s %13s %14s %16s", "---------",
+        cli_out ("%40s %16s %13s %13s %13s %13s %14s %s", "Node",
+                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                 "status", "run time in secs");
+        cli_out ("%40s %16s %13s %13s %13s %13s %14s %16s", "---------",
                  "-----------", "-----------", "-----------", "-----------",
-                 "------------", "--------------");
+                 "-----------", "------------", "--------------");
         do {
                 snprintf (key, 256, "node-uuid-%d", i);
                 ret = dict_get_str (dict, key, &node_uuid);
@@ -1402,6 +1404,12 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 "failed to get failures count");
 
                 memset (key, 0, 256);
+                snprintf (key, 256, "skipped-%d", i);
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get skipped count");
+                memset (key, 0, 256);
                 snprintf (key, 256, "run-time-%d", i);
                 ret = dict_get_double (dict, key, &elapsed);
                 if (ret)
@@ -1411,8 +1419,8 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 status = cli_vol_task_status_str[status_rcd];
                 size_str = gf_uint64_2human_readable(size);
                 cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64 " %13"PRIu64
-                         " %14s %16.2f", node_uuid, files, size_str, lookup,
-                         failures, status, elapsed);
+                         " %13"PRIu64 " %14s %16.2f", node_uuid, files,
+                         size_str, lookup, failures, skipped, status, elapsed);
                 GF_FREE(size_str);
 
                 i++;
@@ -1724,6 +1732,7 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
         char                    *node_uuid = 0;
         gf_defrag_status_t       status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
         uint64_t                 failures = 0;
+        uint64_t                 skipped = 0;
         double                   elapsed = 0;
         char                    *size_str = NULL;
 
@@ -1798,11 +1807,12 @@ xml_output:
         }
 
 
-        cli_out ("%40s %16s %13s %13s %13s %14s %s", "Node", "Rebalanced-files",
-                 "size", "scanned", "failures", "status", "run-time in secs");
-        cli_out ("%40s %16s %13s %13s %13s %14s %16s", "---------",
+        cli_out ("%40s %16s %13s %13s %13s %13s %14s %s", "Node",
+                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                 "status", "run-time in secs");
+        cli_out ("%40s %16s %13s %13s %13s %13s %14s %16s", "---------",
                  "-----------", "-----------", "-----------", "-----------",
-                 "------------", "--------------");
+                  "-----------","------------", "--------------");
 
         do {
                 snprintf (key, 256, "node-uuid-%d", i);
@@ -1845,6 +1855,11 @@ xml_output:
                         gf_log (THIS->name, GF_LOG_TRACE,
                                 "Failed to get failure on files");
 
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "Failed to get skipped files");
                 memset (key, 0, 256);
                 snprintf (key, 256, "run-time-%d", i);
                 ret = dict_get_double (dict, key, &elapsed);
diff --git a/tests/volume.rc b/tests/volume.rc
index 044333a83b4..d942e485fa5 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -27,11 +27,11 @@ function volume_option()
 }
 
 function rebalance_status_field {
-        $CLI volume rebalance $1 status | awk '{print $6}' | sed -n 3p
+        $CLI volume rebalance $1 status | awk '{print $7}' | sed -n 3p
 }
 
 function remove_brick_status_completed_field {
-        $CLI volume remove-brick $V0 $H0:$B0/r2d2_{4,5} status | awk '{print $6}' | sed -n 3p
+        $CLI volume remove-brick $V0 $H0:$B0/r2d2_{4,5} status | awk '{print $7}' | sed -n 3p
 }
 
 function get_mount_process_pid {
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 980a385b19b..e220f554c00 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -218,6 +218,7 @@ struct gf_defrag_info_ {
         uint64_t                     total_data;
         uint64_t                     num_files_lookedup;
         uint64_t                     total_failures;
+        uint64_t                     skipped;
         gf_lock_t                    lock;
         int                          cmd;
         pthread_t                    th;
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 78c0340bdf4..b28b9e8f358 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -1064,6 +1064,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         struct timeval           end            = {0,};
         double                   elapsed        = {0,};
         struct timeval           start          = {0,};
+        int32_t                  err            = 0;
 
         gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
                 loc->path);
@@ -1215,9 +1216,21 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         ret = syncop_setxattr (this, &entry_loc, migrate_data,
                                                0);
                         if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "migrate-data"
-                                        " failed for %s", entry_loc.path);
-                                defrag->total_failures +=1;
+                                err = op_errno;
+                                /* errno is overloaded. See
+                                 * rebalance_task_completion () */
+                                if (err != ENOSPC) {
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data skipped for %s"
+                                                " due to space constraints",
+                                                entry_loc.path);
+                                        defrag->skipped +=1;
+                                } else{
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "migrate-data failed for %s",
+                                                entry_loc.path);
+                                        defrag->total_failures +=1;
+                                }
                         }
 
                         if (ret == -1) {
@@ -1616,6 +1629,7 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         uint64_t size   = 0;
         uint64_t lookup = 0;
         uint64_t failures = 0;
+        uint64_t skipped = 0;
         char     *status = "";
         double   elapsed = 0;
         struct timeval end = {0,};
@@ -1632,6 +1646,7 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         size   = defrag->total_data;
         lookup = defrag->num_files_lookedup;
         failures = defrag->total_failures;
+        skipped = defrag->skipped;
 
         gettimeofday (&end, NULL);
 
@@ -1655,6 +1670,7 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "failed to set lookedup file count");
 
+
         ret = dict_set_int32 (dict, "status", defrag->defrag_status);
         if (ret)
                 gf_log (THIS->name, GF_LOG_WARNING,
@@ -1667,6 +1683,14 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         }
 
         ret = dict_set_uint64 (dict, "failures", failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set failure count");
+
+        ret = dict_set_uint64 (dict, "skipped", skipped);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set skipped file count");
 log:
         switch (defrag->defrag_status) {
         case GF_DEFRAG_STATUS_NOT_STARTED:
@@ -1689,8 +1713,8 @@ log:
         gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s. Time taken is %.2f "
                 "secs", status, elapsed);
         gf_log (THIS->name, GF_LOG_INFO, "Files migrated: %"PRIu64", size: %"
-                PRIu64", lookups: %"PRIu64", failures: %"PRIu64, files, size,
-                lookup, failures);
+                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
+                "%"PRIu64, files, size, lookup, failures, skipped);
 
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index ad5765d7b77..de17c3dbf1f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -6170,6 +6170,7 @@ glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo)
         rebal->lookedup_files = 0;
         rebal->rebalance_failures = 0;
         rebal->rebalance_time = 0;
+        rebal->skipped_files = 0;
 
 }
 
@@ -6266,6 +6267,7 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
         uint64_t                        lookup = 0;
         gf_defrag_status_t              status = GF_DEFRAG_STATUS_NOT_STARTED;
         uint64_t                        failures = 0;
+        uint64_t                        skipped = 0;
         xlator_t                       *this = NULL;
         double                          run_time = 0;
 
@@ -6296,6 +6298,11 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
                 gf_log (this->name, GF_LOG_TRACE,
                         "failed to get failure count");
 
+        ret = dict_get_uint64 (rsp_dict, "skipped", &skipped);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get skipped count");
+
         ret = dict_get_double (rsp_dict, "run-time", &run_time);
         if (ret)
                 gf_log (this->name, GF_LOG_TRACE,
@@ -6311,6 +6318,8 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
                 volinfo->rebal.defrag_status = status;
         if (failures)
                 volinfo->rebal.rebalance_failures = failures;
+        if (skipped)
+                volinfo->rebal.skipped_files = skipped;
         if (run_time)
                 volinfo->rebal.rebalance_time = run_time;
 
@@ -7040,6 +7049,18 @@ glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
         }
 
         memset (key, 0, 256);
+        snprintf (key, 256, "skipped-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "skipped-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set skipped count");
+                }
+        }
+        memset (key, 0, 256);
         snprintf (key, 256, "run-time-%d", index);
         ret = dict_get_double (rsp_dict, key, &elapsed_time);
         if (!ret) {
@@ -7403,6 +7424,13 @@ glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
                 gf_log (THIS->name, GF_LOG_ERROR,
                         "failed to set failure count");
 
+        memset (key, 0 , 256);
+        snprintf (key, 256, "skipped-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.skipped_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set skipped count");
+
         memset (key, 0, 256);
         snprintf (key, 256, "run-time-%d", i);
         ret = dict_set_double (op_ctx, key, volinfo->rebal.rebalance_time);
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index a4795a03136..00331258e97 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -234,6 +234,7 @@ struct glusterd_rebalance_ {
         uint64_t                rebalance_files;
         uint64_t                rebalance_data;
         uint64_t                lookedup_files;
+        uint64_t                skipped_files;
         glusterd_defrag_info_t  *defrag;
         gf_cli_defrag_type      defrag_cmd;
         uint64_t                rebalance_failures;