dht: Rebalance status should display status from each process

Change-Id: Id128417219bdb7146253618a5f8f31ef35013894
BUG: 801322
Signed-off-by: shishir gowda <shishirng@gluster.com>
Reviewed-on: http://review.gluster.com/2942
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>

5 files changed, 189 insertions, 90 deletions

diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index c47c08e3d18..4be25c9e4e0 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -1009,6 +1009,10 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
         uint64_t                 lookup  = 0;
         char                     msg[1024] = {0,};
         gf_defrag_status_t       status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+        int32_t                  counter = 0;
+        char                    *node_uuid = NULL;
+        char                     key[256] = {0,};
+        int32_t                  i = 1;
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1059,25 +1063,23 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 }
         }
 
-        ret = dict_get_uint64 (dict, "files", &files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get file count");
-
-        ret = dict_get_uint64 (dict, "size", &size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get size of xfer");
-
-        ret = dict_get_uint64 (dict, "lookups", &lookup);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get lookedup file count");
+        if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS))) {
+                /* All other possibility is about starting a volume */
+                if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
+                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                else
+                        snprintf (msg, sizeof (msg),
+                                  "Starting rebalance on volume %s has been %s",
+                                  volname, (rsp.op_ret) ? "unsuccessful":
+                                  "successful");
+                goto done;
+        }
 
-        ret = dict_get_int32 (dict, "status", (int32_t *)&status_rcd);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get status");
+        ret = dict_get_int32 (dict, "count", &counter);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "count not set");
+                goto out;
+        }
 
         if (cmd == GF_DEFRAG_CMD_STOP) {
                 if (rsp.op_ret == -1) {
@@ -1088,13 +1090,12 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 snprintf (msg, sizeof (msg),
                                           "rebalance volume %s stop failed",
                                           volname);
+                        goto done;
                 } else {
                         snprintf (msg, sizeof (msg),
-                                 "Stopped rebalance process on volume %s \n"
-                                 "(after rebalancing %"PRId64" bytes - "
-                                 "%"PRId64" files)", volname, size, files);
+                                 "Stopped rebalance process on volume %s \n",
+                                  volname);
                 }
-                goto done;
         }
         if (cmd == GF_DEFRAG_CMD_STATUS) {
                 if (rsp.op_ret == -1) {
@@ -1107,6 +1108,42 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                           "rebalance process");
                         goto done;
                 }
+        }
+        cli_out ("%40s %16s %13s %13s %14s", "Node", "Rebalanced-files",
+                 "size", "scanned", "status");
+        cli_out ("%40s %16s %13s %13s %14s", "---------", "-----------",
+                 "-----------", "-----------", "------------");
+
+        do {
+                snprintf (key, 256, "node-uuid-%d", i);
+                ret = dict_get_str (dict, key, &node_uuid);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "failed to get node-uuid");
+
+                snprintf (key, 256, "files-%d", i);
+                ret = dict_get_uint64 (dict, key, &files);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "failed to get file count");
+
+                snprintf (key, 256, "size-%d", i);
+                ret = dict_get_uint64 (dict, key, &size);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "failed to get size of xfer");
+
+                snprintf (key, 256, "lookups-%d", i);
+                ret = dict_get_uint64 (dict, key, &lookup);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "failed to get lookedup file count");
+
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "failed to get status");
 
                 switch (status_rcd) {
                 case GF_DEFRAG_STATUS_NOT_STARTED:
@@ -1125,27 +1162,11 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         status = "failed";
                         break;
                 }
-                if (files || size || lookup) {
-                        snprintf (msg, sizeof(msg),
-                                  "Rebalance %s: rebalanced %"PRId64
-                                  " files of size %"PRId64" (total files"
-                                  " scanned %"PRId64")", status,
-                                  files, size, lookup);
-                        goto done;
-                }
-
-                snprintf (msg, sizeof (msg), "Rebalance %s", status);
-                goto done;
-        }
+                cli_out ("%40s %16"PRId64 "%13"PRId64 "%13"PRId64 "%14s", node_uuid, files,
+                        size, lookup, status);
+                i++;
+        } while (i <= counter);
 
-        /* All other possibility is about starting a volume */
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg),
-                          "Starting rebalance on volume %s has been %s",
-                          volname, (rsp.op_ret) ? "unsuccessful":
-                          "successful");
 
 done:
 #if (HAVE_LIB_XML)
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 46c02bff711..f9813db12f5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -2909,7 +2909,13 @@ glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
         uint64_t                        size = 0;
         uint64_t                        lookup = 0;
         gf_defrag_status_t              status = GF_DEFRAG_STATUS_NOT_STARTED;
+        char                            key[256] = {0,};
+        int32_t                         i = 0;
+        char                            buf[1024] = {0,};
+        char                            *node_str = NULL;
+        glusterd_conf_t                 *priv = NULL;
 
+        priv = THIS->private;
         GF_ASSERT (req_dict);
 
         ret = dict_get_str (req_dict, "volname", &volname);
@@ -2923,6 +2929,9 @@ glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
         if (ret)
                 goto out;
 
+        if (!rsp_dict)
+                goto populate;
+
         ret = dict_get_uint64 (rsp_dict, "files", &files);
         if (ret)
                 gf_log (THIS->name, GF_LOG_TRACE,
@@ -2942,30 +2951,61 @@ glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
                 gf_log (THIS->name, GF_LOG_TRACE,
                         "failed to get status");
 
-        volinfo->rebalance_files += files;
-        volinfo->rebalance_data += size;
-        volinfo->lookedup_files += lookup;
+        if (files)
+                volinfo->rebalance_files = files;
+        if (size)
+                volinfo->rebalance_data = size;
+        if (lookup)
+                volinfo->lookedup_files = lookup;
 
         if (!op_ctx) {
                 dict_copy (rsp_dict, op_ctx);
                 goto out;
         }
 
-        ret = dict_set_uint64 (op_ctx, "files", volinfo->rebalance_files);
+populate:
+        ret = dict_get_int32 (op_ctx, "count", &i);
+        i++;
+
+        ret = dict_set_int32 (op_ctx, "count", i);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set count");
+
+        snprintf (buf, 1024, "%s", uuid_utoa (priv->uuid));
+        node_str = gf_strdup (buf);
+
+        snprintf (key, 256, "node-uuid-%d",i);
+        ret = dict_set_dynstr (op_ctx, key, node_str);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set node-uuid");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "files-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebalance_files);
         if (ret)
                 gf_log (THIS->name, GF_LOG_ERROR,
                         "failed to set file count");
 
-        ret = dict_set_uint64 (op_ctx, "size", volinfo->rebalance_data);
+        memset (key, 0 , 256);
+        snprintf (key, 256, "size-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebalance_data);
         if (ret)
                 gf_log (THIS->name, GF_LOG_ERROR,
                         "failed to set size of xfer");
 
-        ret = dict_set_uint64 (op_ctx, "lookups", volinfo->lookedup_files);
+        memset (key, 0 , 256);
+        snprintf (key, 256, "lookups-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->lookedup_files);
         if (ret)
                 gf_log (THIS->name, GF_LOG_ERROR,
                         "failed to set lookedup file count");
-        ret = dict_set_int32 (op_ctx, "status", status);
+        if (!status)
+                status = volinfo->defrag_status;
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "status-%d", i);
+        ret = dict_set_int32 (op_ctx, key, status);
         if (ret)
                 gf_log (THIS->name, GF_LOG_ERROR,
                         "failed to set status");
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index cc2eacffd35..bfc41b29252 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -275,4 +275,8 @@ glusterd_gsync_get_param_file (char *prmfile, const char *ext, char *master,
                                 char *slave, char *gl_workdir);
 int
 glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag);
+
+int
+glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                 dict_t *op_ctx);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 936a3b26e6c..5d567e37a8a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -516,31 +516,8 @@ glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
                 }
                 break;
         case GF_DEFRAG_CMD_STATUS:
-                ret = glusterd_is_defrag_on (volinfo);
-                if (!ret) {
-                        ret = -1;
-                        if (volinfo->defrag_status ==
-                                GF_DEFRAG_STATUS_COMPLETE) {
-                                snprintf (msg, sizeof (msg), "Rebalance "
-                                          "completed!");
-                                goto out;
-                        }
-                        snprintf (msg, sizeof(msg), "Rebalance is not running"
-                                  " on volume %s", volname);
-                        goto out;
-                }
-                break;
-
         case GF_DEFRAG_CMD_STOP:
-                ret = glusterd_is_defrag_on (volinfo);
-                if (!ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "rebalance is not running");
-                        ret = -1;
-                        snprintf (msg, sizeof(msg), "Rebalance is not running"
-                                  " on volume %s", volname);
-                        goto out;
-                }
+                break;
         default:
                 break;
         }
@@ -594,9 +571,6 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                  break;
         case GF_DEFRAG_CMD_STOP:
         case GF_DEFRAG_CMD_STATUS:
-                volinfo->rebalance_files = 0;
-                volinfo->rebalance_data = 0;
-                volinfo->lookedup_files = 0;
                 break;
         default:
                 break;
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index c4e9afaacef..c7931dbfafc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -1090,6 +1090,11 @@ glusterd_volume_rebalance_use_rsp_dict (dict_t *rsp_dict)
         int32_t        value32  = 0;
         char          *volname  = NULL;
         glusterd_volinfo_t *volinfo = NULL;
+        char           key[256] = {0,};
+        int32_t        index    = 0;
+        int32_t        i        = 0;
+        char          *node_uuid = NULL;
+        char          *node_uuid_str = NULL;
 
         GF_ASSERT (rsp_dict);
 
@@ -1102,6 +1107,12 @@ glusterd_volume_rebalance_use_rsp_dict (dict_t *rsp_dict)
         if (!ctx_dict)
                 goto out;
 
+        ret = dict_get_int32 (ctx_dict, "count", &i);
+        i++;
+        ret = dict_set_int32 (ctx_dict, "count", i);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Failed to set index");
+
         ret = dict_get_str (ctx_dict, "volname", &volname);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
@@ -1113,48 +1124,76 @@ glusterd_volume_rebalance_use_rsp_dict (dict_t *rsp_dict)
         if (ret)
                 goto out;
 
-        ret = dict_get_uint64 (rsp_dict, "files", &value);
+        ret = dict_get_int32 (rsp_dict, "count", &index);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "failed to get index");
+
+        snprintf (key, 256, "files-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
         if (!ret) {
-                volinfo->rebalance_files += value;
-                ret = dict_set_uint64 (ctx_dict, "files",
-                                       volinfo->rebalance_files);
+                memset (key, 0, 256);
+                snprintf (key, 256, "files-%d", i);
+                ret = dict_set_uint64 (ctx_dict, key, value);
                 if (ret) {
                         gf_log (THIS->name, GF_LOG_DEBUG,
                                 "failed to set the file count");
                 }
         }
 
-        ret = dict_get_uint64 (rsp_dict, "size", &value);
+        memset (key, 0, 256);
+        snprintf (key, 256, "size-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
         if (!ret) {
-                volinfo->rebalance_data += value;
-                ret = dict_set_uint64 (ctx_dict, "size",
-                                       volinfo->rebalance_data);
+                memset (key, 0, 256);
+                snprintf (key, 256, "size-%d", i);
+                ret = dict_set_uint64 (ctx_dict, key, value);
                 if (ret) {
                         gf_log (THIS->name, GF_LOG_DEBUG,
                                 "failed to set the size of migration");
                 }
         }
 
-        ret = dict_get_uint64 (rsp_dict, "lookups", &value);
+        memset (key, 0, 256);
+        snprintf (key, 256, "lookups-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
         if (!ret) {
-                volinfo->lookedup_files += value;
-                ret = dict_set_uint64 (ctx_dict, "lookups",
-                                       volinfo->lookedup_files);
+                memset (key, 0, 256);
+                snprintf (key, 256, "lookups-%d", i);
+                ret = dict_set_uint64 (ctx_dict, key, value);
                 if (ret) {
                         gf_log (THIS->name, GF_LOG_DEBUG,
                                 "failed to set lookuped file count");
                 }
         }
 
-        ret = dict_get_int32 (rsp_dict, "status", &value32);
+        memset (key, 0, 256);
+        snprintf (key, 256, "status-%d", index);
+        ret = dict_get_int32 (rsp_dict, key, &value32);
         if (!ret) {
-                ret = dict_set_int32 (ctx_dict, "status", value32);
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_set_int32 (ctx_dict, key, value32);
                 if (ret) {
                         gf_log (THIS->name, GF_LOG_DEBUG,
                                 "failed to set status");
                 }
         }
 
+        memset (key, 0, 256);
+        snprintf (key, 256, "node-uuid-%d", index);
+        ret = dict_get_str (rsp_dict, key, &node_uuid);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "node-uuid-%d", i);
+                node_uuid_str = gf_strdup (node_uuid);
+                ret = dict_set_dynstr (ctx_dict, key, node_uuid_str);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set node-uuid");
+                }
+        }
+        ret = 0;
+
 out:
         return ret;
 }
@@ -1829,6 +1868,7 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
         glusterd_pending_node_t         *pending_node;
         glusterd_req_ctx_t              *req_ctx = NULL;
         struct rpc_clnt                 *rpc = NULL;
+        dict_t                          *op_ctx = NULL;
 
         if (!this) {
                 ret = -1;
@@ -1872,6 +1912,26 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
 
                 rpc = glusterd_pending_node_get_rpc (pending_node);
                 if (!rpc) {
+                        if (pending_node->type == GD_NODE_REBALANCE) {
+                                opinfo.brick_pending_count = 0;
+                                ret = 0;
+                                if (req) {
+                                        if (req->input.input_val)
+                                                GF_FREE (req->input.input_val);
+                                        GF_FREE (req);
+                                        req = NULL;
+                                }
+                                GLUSTERD_STACK_DESTROY (dummy_frame);
+
+                                op_ctx = glusterd_op_get_ctx ();
+                                if (!op_ctx)
+                                        goto out;
+                                glusterd_defrag_volume_node_rsp (req_ctx->dict,
+                                                                 NULL, op_ctx);
+
+                                goto out;
+                        }
+
                         ret = -1;
                         gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
                                 "due to rpc failure.");