dht/rebalance: Display migration failure count

Change-Id: Ib2f8be2c98eae1b4df5d2090c1f1eec8ab264a35
BUG: 811923
Signed-off-by: shishir gowda <shishirng@gluster.com>
Reviewed-on: http://review.gluster.com/3135
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

7 files changed, 71 insertions, 14 deletions

diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index a7450c32e60..4a1ff22b8ef 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -1016,6 +1016,7 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                    *node_uuid = NULL;
         char                     key[256] = {0,};
         int32_t                  i = 1;
+        uint64_t                 failures = 0;
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1114,10 +1115,10 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         goto done;
                 }
         }
-        cli_out ("%40s %16s %13s %13s %14s", "Node", "Rebalanced-files",
-                 "size", "scanned", "status");
-        cli_out ("%40s %16s %13s %13s %14s", "---------", "-----------",
-                 "-----------", "-----------", "------------");
+        cli_out ("%40s %16s %13s %13s %13s %14s", "Node", "Rebalanced-files",
+                 "size", "scanned", "failures","status");
+        cli_out ("%40s %16s %13s %13s %13s %14s", "---------", "-----------",
+                 "-----------", "-----------", "-----------", "------------");
 
         do {
                 snprintf (key, 256, "node-uuid-%d", i);
@@ -1154,6 +1155,13 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         gf_log (THIS->name, GF_LOG_TRACE,
                                 "failed to get status");
 
+                memset (key, 0, 256);
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "failed to get failures count");
+
                 switch (status_rcd) {
                 case GF_DEFRAG_STATUS_NOT_STARTED:
                         status = "not started";
@@ -1171,8 +1179,9 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         status = "failed";
                         break;
                 }
-                cli_out ("%40s %16"PRId64 "%13"PRId64 "%13"PRId64 "%14s", node_uuid, files,
-                        size, lookup, status);
+                cli_out ("%40s %16"PRId64 "%13"PRId64 "%13"PRId64 "%13"PRId64
+                         " %14s", node_uuid, files, size, lookup, failures,
+                         status);
                 i++;
         } while (i <= counter);
 
@@ -1421,6 +1430,7 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
         int32_t                  counter = 0;
         char                    *node_uuid = 0;
         gf_defrag_status_t       status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+        uint64_t                 failures = 0;
 
 
         if (-1 == req->rpc_status) {
@@ -1466,10 +1476,10 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
 
-        cli_out ("%40s %16s %13s %13s %14s", "Node", "Rebalanced-files",
-                 "size", "scanned", "status");
-        cli_out ("%40s %16s %13s %13s %14s", "---------", "-----------",
-                 "-----------", "-----------", "------------");
+        cli_out ("%40s %16s %13s %13s %13s %14s", "Node", "Rebalanced-files",
+                 "size", "scanned", "failures", "status");
+        cli_out ("%40s %16s %13s %13s %13s %14s", "---------", "-----------",
+                 "-----------", "-----------", "-----------", "------------");
 
         do {
                 snprintf (key, 256, "node-uuid-%d", i);
@@ -1506,6 +1516,12 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
                         gf_log (THIS->name, GF_LOG_TRACE,
                                 "failed to get status");
 
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "Failed to get failure on files");
+
                 switch (status_rcd) {
                 case GF_DEFRAG_STATUS_NOT_STARTED:
                         status = "not started";
@@ -1523,8 +1539,9 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
                         status = "failed";
                         break;
                 }
-                cli_out ("%40s %16"PRId64 "%13"PRId64 "%13"PRId64 "%14s", node_uuid, files,
-                        size, lookup, status);
+                cli_out ("%40s %16"PRId64 "%13"PRId64 "%13"PRId64 "%13"PRId64
+                        " %14s", node_uuid, files, size, lookup, failures,
+                        status);
                 i++;
         } while (i <= counter);
 
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 61687aacca0..cf7b93c1ed3 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -204,6 +204,7 @@ struct gf_defrag_info_ {
         uint64_t                     total_files;
         uint64_t                     total_data;
         uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
         gf_lock_t                    lock;
         int                          cmd;
         pthread_t                    th;
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 5e46a80a127..819ec1daeb6 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -1202,9 +1202,11 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
 
                         ret = syncop_setxattr (this, &entry_loc, migrate_data,
                                                0);
-                        if (ret)
+                        if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "migrate-data"
                                         " failed for %s", entry_loc.path);
+                                defrag->total_failures +=1;
+                        }
 
                         if (ret == -1) {
                                 op_errno = errno;
@@ -1548,6 +1550,7 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         uint64_t files  = 0;
         uint64_t size   = 0;
         uint64_t lookup = 0;
+        uint64_t failures = 0;
 
         if (!defrag)
                 goto out;
@@ -1559,6 +1562,7 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         files  = defrag->total_files;
         size   = defrag->total_data;
         lookup = defrag->num_files_lookedup;
+        failures = defrag->total_failures;
 
         if (!dict)
                 goto log;
@@ -1582,9 +1586,12 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         if (ret)
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "failed to set status");
+
+        ret = dict_set_uint64 (dict, "failures", failures);
 log:
         gf_log (THIS->name, GF_LOG_INFO, "Files migrated: %"PRIu64", size: %"
-                PRIu64", lookups: %"PRIu64, files, size, lookup);
+                PRIu64", lookups: %"PRIu64", failures: %"PRIu64, files, size,
+                lookup, failures);
 
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 1825065a4b4..6d646c111f2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -3187,6 +3187,7 @@ glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
         char                            buf[1024] = {0,};
         char                            *node_str = NULL;
         glusterd_conf_t                 *priv = NULL;
+        uint64_t                        failures = 0;
 
         priv = THIS->private;
         GF_ASSERT (req_dict);
@@ -3224,12 +3225,19 @@ glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
                 gf_log (THIS->name, GF_LOG_TRACE,
                         "failed to get status");
 
+        ret = dict_get_uint64 (rsp_dict, "failures", &failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_TRACE,
+                        "failed to get failure count");
+
         if (files)
                 volinfo->rebalance_files = files;
         if (size)
                 volinfo->rebalance_data = size;
         if (lookup)
                 volinfo->lookedup_files = lookup;
+        if (failures)
+                volinfo->rebalance_failures = failures;
 
         if (!op_ctx) {
                 dict_copy (rsp_dict, op_ctx);
@@ -3283,6 +3291,13 @@ populate:
                 gf_log (THIS->name, GF_LOG_ERROR,
                         "failed to set status");
 
+        memset (key, 0 , 256);
+        snprintf (key, 256, "failures-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebalance_failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set failure count");
+
 out:
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index e3c4c47411f..fddf6699f9e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -290,6 +290,7 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
         volinfo->rebalance_files = 0;
         volinfo->rebalance_data = 0;
         volinfo->lookedup_files = 0;
+        volinfo->rebalance_failures = 0;
 
         volinfo->defrag_cmd = cmd;
         glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 14bcf8b9904..ddc4d1f492d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -1208,6 +1208,20 @@ glusterd_volume_rebalance_use_rsp_dict (dict_t *rsp_dict)
                                 "failed to set node-uuid");
                 }
         }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "failures-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set failure count");
+                }
+        }
+
         ret = 0;
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 8169562d78b..1b958d105d6 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -166,6 +166,7 @@ struct glusterd_defrag_info_ {
         uint64_t                     total_files;
         uint64_t                     total_data;
         uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
         gf_lock_t                    lock;
         int                          cmd;
         pthread_t                    th;
@@ -226,6 +227,7 @@ struct glusterd_volinfo_ {
         uint64_t                lookedup_files;
         glusterd_defrag_info_t  *defrag;
         gf_cli_defrag_type      defrag_cmd;
+        uint64_t                rebalance_failures;
 
         /* Replace brick status */
         gf_rb_status_t          rb_status;