glusterd: Fix to log command status at the appropriate time

PROBLEM:
In the existing implementation, the success/failure of
execution of a command is decided (and logged) in glusterd
handler functions. Strictly speaking, the logging mechanism
must take into account what course the command takes within
the state machine before concluding whether it succeeded or
failed.

FIX:
This patch attempts to fix the above issue for vol commands.
The format of the log message is as follows:
    for failure:
    <command string> : FAILED : <cause of failure>
    for success:
    <command string> : SUCCESS

APPROACH (in a nutshell):

 * The command string is packed into dict at cli and sent to
   glusterd.
 * glusterd logs the command status just before doing a
   "submit_reply", which is called (either directly or
   indirectly via a call to glusterd_op_cli_send_response)
   at 2 places for every vol command:
   i. in handler functions, and
   ii. in glusterd_op_txn_complete

In short, the failure of a command in the handler implies the
command has indeed failed. However, its success in the handler
does NOT necessarily mean the command succeeded/will succeed.

Change-Id: I5a8a2ddc318ef2dc2a9699f704a6bcd2f0ab0277
BUG: 823081
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: http://review.gluster.org/3948
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

3 files changed, 237 insertions, 362 deletions

diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 3a67de131..b9bed82e1 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -29,6 +29,17 @@
 #include "cli1-xdr.h"
 #include "run.h"
 
+#define SAVE_WORDS_IN_LOCAL(local, words, frame)            \
+        do {                                                \
+                local = cli_local_get ();                   \
+                                                            \
+                if (local) {                                \
+                        local->words = words;               \
+                        if (frame)                          \
+                                frame->local = local;       \
+                }                                           \
+        } while (0)
+
 extern struct rpc_clnt *global_rpc;
 
 extern rpc_clnt_prog_t *cli_rpc_prog;
@@ -111,6 +122,7 @@ cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     sent = 0;
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
+        cli_local_t             *local = NULL;
 
         if ((wordcount < 3) || (wordcount > 4)) {
                 cli_usage_out (word->pattern);
@@ -151,6 +163,8 @@ cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -332,6 +346,7 @@ cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int32_t                 brick_count = 0;
         int32_t                 sub_count = 0;
         int32_t                 type = GF_CLUSTER_TYPE_NONE;
+        cli_local_t             *local = NULL;
 
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
@@ -340,6 +355,8 @@ cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_volume_create_parse (words, wordcount, &options);
 
         if (ret) {
@@ -409,6 +426,7 @@ cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
         const char              *question = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         question = "Deleting volume will erase all information about the volume. "
                    "Do you want to continue?";
@@ -418,6 +436,8 @@ cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         if (wordcount != 3) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
@@ -460,6 +480,7 @@ cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
         int                     flags = 0;
+        cli_local_t             *local = NULL;
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
@@ -471,6 +492,8 @@ cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
                goto out;
         }
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         dict = dict_new ();
         if (!dict) {
                 goto out;
@@ -583,6 +606,7 @@ cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
         char                    *volname = NULL;
+        cli_local_t             *local = NULL;
 
         const char *question = "Stopping volume will make its data inaccessible. "
                                "Do you want to continue?";
@@ -591,6 +615,8 @@ cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         if (wordcount < 3 || wordcount > 4) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
@@ -718,6 +744,7 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t               *dict = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t          *local = NULL;
 #ifdef GF_SOLARIS_HOST_OS
         cli_out ("Command not supported on Solaris");
         goto out;
@@ -727,6 +754,8 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         dict = dict_new ();
         if (!dict)
                 goto out;
@@ -813,6 +842,7 @@ cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
         rpc_clnt_procedure_t    *proc = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
 
@@ -820,6 +850,8 @@ cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_volume_reset_parse (words, wordcount, &options);
 
         if (ret) {
@@ -859,6 +891,7 @@ cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
         rpc_clnt_procedure_t    *proc    = NULL;
         call_frame_t            *frame   = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
 
         ret = cli_cmd_volume_profile_parse (words, wordcount, &options);
 
@@ -874,6 +907,8 @@ cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
@@ -905,6 +940,7 @@ cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
         rpc_clnt_procedure_t    *proc = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
 
@@ -912,6 +948,8 @@ cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_volume_set_parse (words, wordcount, &options);
 
         if (ret) {
@@ -952,6 +990,7 @@ cli_cmd_volume_add_brick_cbk (struct cli_state *state,
         int                     sent = 0;
         int                     parse_error = 0;
         gf_answer_t             answer = GF_ANSWER_NO;
+        cli_local_t             *local = NULL;
 
         const char *question = "Changing the 'stripe count' of the volume is "
                 "not a supported feature. In some cases it may result in data "
@@ -963,6 +1002,8 @@ cli_cmd_volume_add_brick_cbk (struct cli_state *state,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options);
 
         if (ret) {
@@ -1015,6 +1056,7 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
         call_frame_t            *frame     = NULL;
         dict_t                  *options   = NULL;
         gf_answer_t              answer    = GF_ANSWER_NO;
+        cli_local_t             *local     = NULL;
         const char *question = "Disabling quota will delete all the quota "
                                "configuration. Do you want to continue?";
 
@@ -1030,6 +1072,8 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 goto out;
         }
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_quota_parse (words, wordcount, &options);
         if (ret < 0) {
                 cli_usage_out (word->pattern);
@@ -1071,6 +1115,7 @@ cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
         int                     sent = 0;
         int                     parse_error = 0;
         int                     need_question = 0;
+        cli_local_t             *local = NULL;
 
         const char *question = "Removing brick(s) can result in data loss. "
                                "Do you want to Continue?";
@@ -1079,6 +1124,8 @@ cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_volume_remove_brick_parse (words, wordcount, &options,
                                                  &need_question);
 
@@ -1131,6 +1178,7 @@ cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
         dict_t                  *options = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
 #ifdef GF_SOLARIS_HOST_OS
         cli_out ("Command not supported on Solaris");
@@ -1142,6 +1190,8 @@ cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_volume_replace_brick_parse (words, wordcount, &options);
 
         if (ret) {
@@ -1190,6 +1240,7 @@ cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t                  *options = NULL;
         int                     sent     = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         ret = cli_cmd_volume_top_parse (words, wordcount, &options);
 
@@ -1205,6 +1256,8 @@ cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
@@ -1236,6 +1289,7 @@ cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t                  *options = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         if (!((wordcount == 4) || (wordcount == 5))) {
                 cli_usage_out (word->pattern);
@@ -1249,6 +1303,8 @@ cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_log_rotate_parse (words, wordcount, &options);
         if (ret)
                 goto out;
@@ -1259,14 +1315,13 @@ cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
 
 out:
         if (options)
-                dict_destroy (options);
+                dict_unref (options);
 
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume log rotate failed");
         }
-
         CLI_STACK_DESTROY (frame);
 
         return ret;
@@ -1341,6 +1396,7 @@ cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word
         dict_t                  *options = NULL;
         rpc_clnt_procedure_t    *proc    = NULL;
         call_frame_t            *frame   = NULL;
+        cli_local_t             *local   = NULL;
 
         proc = &cli_rpc_prog->proctable [GLUSTER_CLI_GSYNC_SET];
         if (proc == NULL) {
@@ -1354,6 +1410,8 @@ cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word
                 goto out;
         }
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = cli_cmd_gsync_set_parse (words, wordcount, &options);
         if (ret) {
                 cli_usage_out (word->pattern);
@@ -1386,6 +1444,7 @@ cli_cmd_volume_status_cbk (struct cli_state *state,
         call_frame_t         *frame       = NULL;
         dict_t               *dict        = NULL;
         uint32_t              cmd         = 0;
+        cli_local_t          *local       = NULL;
 
         ret = cli_cmd_volume_status_parse (words, wordcount, &dict);
 
@@ -1413,6 +1472,8 @@ cli_cmd_volume_status_cbk (struct cli_state *state,
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         ret = proc->fn (frame, THIS, dict);
 
  out:
@@ -1612,12 +1673,15 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     parse_error = 0;
         dict_t                  *options = NULL;
         xlator_t                *this = NULL;
+        cli_local_t             *local = NULL;
 
         this = THIS;
         frame = create_frame (this, this->ctx->pool);
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         if (wordcount < 3) {
                cli_usage_out (word->pattern);
                parse_error = 1;
@@ -1662,11 +1726,14 @@ cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word
         dict_t                          *options = NULL;
         int                             sent = 0;
         int                             parse_error = 0;
+        cli_local_t                     *local = NULL;
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         if (wordcount < 3) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
@@ -1747,11 +1814,14 @@ cli_cmd_volume_clearlocks_cbk (struct cli_state *state,
         dict_t                          *options = NULL;
         int                             sent = 0;
         int                             parse_error = 0;
+        cli_local_t                     *local = NULL;
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
                 goto out;
 
+        SAVE_WORDS_IN_LOCAL (local, words, frame);
+
         if (wordcount < 7 || wordcount > 8) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 868a587f8..368a03cc0 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -67,6 +67,10 @@ int32_t
 gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
                       void *data);
 
+int
+cli_to_glusterd (gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                 xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                 rpc_clnt_prog_t *prog, struct iobref *iobref);
 
 rpc_clnt_prog_t cli_handshake_prog = {
         .progname  = "cli handshake",
@@ -2533,7 +2537,6 @@ gf_cli_create_volume (call_frame_t *frame, xlator_t *this,
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
         dict_t                  *dict = NULL;
-        cli_local_t             *local = NULL;
 
         if (!frame || !this ||  !data) {
                 ret = -1;
@@ -2542,34 +2545,14 @@ gf_cli_create_volume (call_frame_t *frame, xlator_t *this,
 
         dict = dict_ref ((dict_t *)data);
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        local = cli_local_get ();
-
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_CREATE_VOLUME, NULL,
-                              this, gf_cli_create_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
-
-
+        ret = cli_to_glusterd (&req, frame, gf_cli_create_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (dict)
-                dict_unref (dict);
-
         GF_FREE (req.dict.dict_val);
 
         return ret;
@@ -2581,7 +2564,6 @@ gf_cli_delete_volume (call_frame_t *frame, xlator_t *this,
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -2589,35 +2571,19 @@ gf_cli_delete_volume (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        local = cli_local_get ();
-
         dict = dict_new ();
         ret = dict_set_str (dict, "volname", data);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
                 goto out;
         }
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialize dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_DELETE_VOLUME, NULL,
-                              this, gf_cli_delete_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_delete_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-        if (dict)
-                dict_unref (dict);
         GF_FREE (req.dict.dict_val);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
@@ -2630,7 +2596,6 @@ gf_cli_start_volume (call_frame_t *frame, xlator_t *this,
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -2639,26 +2604,11 @@ gf_cli_start_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        local = cli_local_get ();
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize dict");
-                goto out;
-        }
 
-
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_START_VOLUME, NULL,
-                              this, gf_cli_start_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_start_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2672,7 +2622,6 @@ gf_cli_stop_volume (call_frame_t *frame, xlator_t *this,
 {
         gf_cli_req             req = {{0,}};
         int                    ret = 0;
-        cli_local_t            *local = NULL;
         dict_t                 *dict = data;
 
         if (!frame || !this ||  !data) {
@@ -2680,27 +2629,12 @@ gf_cli_stop_volume (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        local = cli_local_get ();
         dict = data;
 
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STOP_VOLUME, NULL,
-                              this, gf_cli_stop_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_stop_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2714,7 +2648,6 @@ gf_cli_defrag_volume (call_frame_t *frame, xlator_t *this,
 {
         gf_cli_req              req     =  {{0,}};
         int                     ret     = 0;
-        cli_local_t            *local   = NULL;
         char                   *volname = NULL;
         char                   *cmd_str = NULL;
         dict_t                 *dict    = NULL;
@@ -2762,7 +2695,6 @@ gf_cli_defrag_volume (call_frame_t *frame, xlator_t *this,
         }
 
 done:
-        local = cli_local_get ();
 
         req_dict = dict_new ();
         if (!req_dict) {
@@ -2784,30 +2716,12 @@ done:
                 goto out;
         }
 
-        if (local) {
-                local->dict = dict_ref (req_dict);
-                frame->local = local;
-                local = NULL;
-        }
-
-        ret = dict_allocate_and_serialize (req_dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_DEFRAG_VOLUME, NULL,
-                              this, gf_cli_defrag_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_defrag_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, req_dict,
+                               GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-        if (local)
-                cli_local_wipe (local);
-
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
@@ -2864,22 +2778,13 @@ gf_cli_reset_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                               GLUSTER_CLI_RESET_VOLUME, NULL,
-                               this, gf_cli_reset_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_reset_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-                gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -2898,18 +2803,10 @@ gf_cli_set_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_SET_VOLUME, NULL,
-                              this, gf_cli_set_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_set_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2944,18 +2841,9 @@ gf_cli_add_brick (call_frame_t *frame, xlator_t *this,
                 goto out;
 
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_ADD_BRICK, NULL,
-                              this, gf_cli_add_brick_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_add_brick_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2977,27 +2865,14 @@ gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
         char                     *volname = NULL;
         dict_t                   *req_dict = NULL;
         int32_t                   cmd = 0;
-        cli_local_t              *local = NULL;
 
         if (!frame || !this ||  !data) {
                 ret = -1;
                 goto out;
         }
 
-        local = cli_local_get ();
-        if (!local) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto out;
-        }
-
-        frame->local = local;
-
         dict = data;
 
-        local->dict = dict_ref (dict);
-
         ret = dict_get_str (dict, "volname", &volname);
         if (ret)
                 goto out;
@@ -3009,18 +2884,11 @@ gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
         if ((command != GF_OP_CMD_STATUS) &&
             (command != GF_OP_CMD_STOP)) {
 
-                ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                                   &req.dict.dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict");
-                        goto out;
-                }
 
-                ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                                      GLUSTER_CLI_REMOVE_BRICK, NULL,
-                                      this, gf_cli_remove_brick_cbk,
-                                      (xdrproc_t) xdr_gf_cli_req);
+                ret = cli_to_glusterd (&req, frame, gf_cli_remove_brick_cbk,
+                                       (xdrproc_t) xdr_gf_cli_req, dict,
+                                       GLUSTER_CLI_REMOVE_BRICK, this,
+                                       cli_rpc_prog, NULL);
         } else {
                 /* Need rebalance status to e sent :-) */
                 req_dict = dict_new ();
@@ -3048,22 +2916,14 @@ gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
                         goto out;
                 }
 
-                ret = dict_allocate_and_serialize (req_dict, &status_req.dict.dict_val,
-                                                   &status_req.dict.dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to serialize the data");
+                ret = cli_to_glusterd (&status_req, frame,
+                                       gf_cli3_remove_brick_status_cbk,
+                                       (xdrproc_t) xdr_gf_cli_req, req_dict,
+                                       GLUSTER_CLI_DEFRAG_VOLUME, this,
+                                       cli_rpc_prog, NULL);
 
-                        goto out;
                 }
 
-                ret = cli_cmd_submit (&status_req, frame, cli_rpc_prog,
-                                      GLUSTER_CLI_DEFRAG_VOLUME, NULL,
-                                      this, gf_cli3_remove_brick_status_cbk,
-                                      (xdrproc_t) xdr_gf_cli_req);
-
-        }
-
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
@@ -3071,9 +2931,6 @@ out:
 
         GF_FREE (status_req.dict.dict_val);
 
-        if (req_dict)
-                dict_unref (req_dict);
-
         return ret;
 }
 
@@ -3083,7 +2940,6 @@ gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
 {
         gf_cli_req                  req        =  {{0,}};
         int                         ret        = 0;
-        cli_local_t                *local      = NULL;
         dict_t                     *dict       = NULL;
         char                       *src_brick  = NULL;
         char                       *dst_brick  = NULL;
@@ -3097,17 +2953,6 @@ gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        local = cli_local_get ();
-        if (!local) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto out;
-        }
-
-        local->dict  = dict_ref (dict);
-        frame->local = local;
-
         ret = dict_get_int32 (dict, "operation", &op);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -3140,19 +2985,10 @@ gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
                 "%s with operation=%d", src_brick,
                 dst_brick, op);
 
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_REPLACE_BRICK, NULL,
-                              this, gf_cli_replace_brick_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_replace_brick_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -3178,18 +3014,10 @@ gf_cli_log_rotate (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to serialize dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_LOG_ROTATE, NULL,
-                              this, gf_cli_log_rotate_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_log_rotate_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog,
+                               NULL);
 
 
 out:
@@ -3213,18 +3041,11 @@ gf_cli_sync_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to serialize dict");
-                goto out;
-        }
 
-        ret = cli_cmd_submit (&req, frame,
-                              cli_rpc_prog, GLUSTER_CLI_SYNC_VOLUME,
-                              NULL, this, gf_cli_sync_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_sync_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -3278,21 +3099,13 @@ gf_cli_quota (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_QUOTA, NULL,
-                              this, gf_cli_quota_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_quota_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_QUOTA, this, cli_rpc_prog, NULL);
 
-        GF_FREE (req.dict.dict_val);
 out:
+        GF_FREE (req.dict.dict_val);
+
         return ret;
 }
 
@@ -3652,19 +3465,10 @@ gf_cli_gsync_set (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GSYNC_SET, NULL,
-                              this, gf_cli_gsync_set_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_gsync_set_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog,
+                               NULL);
 
 out:
         GF_FREE (req.dict.dict_val);
@@ -4021,21 +3825,10 @@ gf_cli_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
                 goto out;
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_PROFILE_VOLUME, NULL,
-                              this, gf_cli_profile_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_profile_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -4295,20 +4088,10 @@ gf_cli_top_volume (call_frame_t *frame, xlator_t *this, void *data)
                 goto out;
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_PROFILE_VOLUME, NULL,
-                              this, gf_cli_top_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_top_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -5588,19 +5371,10 @@ gf_cli_status_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STATUS_VOLUME, NULL,
-                              this, gf_cli_status_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
-
+        ret = cli_to_glusterd (&req, frame, gf_cli_status_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_STATUS_VOLUME, this, cli_rpc_prog,
+                               NULL);
  out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning: %d", ret);
         return ret;
@@ -5624,14 +5398,10 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
         if (ret)
                 goto out;
 
-        local = cli_local_get ();
-        if (!local) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Failed to allocate local");
-                goto out;
+        if (frame->local) {
+                local = frame->local;
+                local->all = _gf_true;
         }
-        frame->local = local;
-        local->all = _gf_true;
 
         ret = gf_cli_status_volume (frame, this, data);
         if (ret)
@@ -6008,7 +5778,6 @@ gf_cli_heal_volume (call_frame_t *frame, xlator_t *this,
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -6017,26 +5786,11 @@ gf_cli_heal_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        local = cli_local_get ();
 
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_HEAL_VOLUME, NULL,
-                              this, gf_cli_heal_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_heal_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_HEAL_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -6103,23 +5857,12 @@ gf_cli_statedump_volume (call_frame_t *frame, xlator_t *this,
 
         options = data;
 
-        ret = dict_allocate_and_serialize (options, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STATEDUMP_VOLUME, NULL,
-                              this, gf_cli_statedump_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_statedump_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, options,
+                               GLUSTER_CLI_STATEDUMP_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-        if (options)
-                dict_destroy (options);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
         GF_FREE (req.dict.dict_val);
@@ -6303,27 +6046,88 @@ gf_cli_clearlocks_volume (call_frame_t *frame, xlator_t *this,
 
         options = data;
 
-        ret = dict_allocate_and_serialize (options, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "failed to serialize the data");
+        ret = cli_to_glusterd (&req, frame, gf_cli_clearlocks_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, options,
+                               GLUSTER_CLI_CLRLOCKS_VOLUME, this, cli_rpc_prog,
+                               NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int
+cli_to_glusterd (gf_cli_req *req, call_frame_t *frame,
+                 fop_cbk_fn_t cbkfn, xdrproc_t xdrproc, dict_t *dict,
+                 int procnum, xlator_t *this, rpc_clnt_prog_t *prog,
+                 struct iobref *iobref)
+{
+        int                ret = 0;
+        size_t             len = 0;
+        char               *cmd = NULL;
+        int                i = 0;
+        const char         **words = NULL;
+        cli_local_t        *local = NULL;
+
+        if (!this || !frame || !dict) {
+                ret = -1;
                 goto out;
         }
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_CLRLOCKS_VOLUME, NULL,
-                              this, gf_cli_clearlocks_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
+        if (!frame->local) {
+                ret = -1;
+                goto out;
+        }
 
-out:
-        if (options)
-                dict_destroy (options);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        local = frame->local;
 
-        GF_FREE (req.dict.dict_val);
+        if (!local->words) {
+                ret = -1;
+                goto out;
+        }
+
+        words = local->words;
+        local->dict = dict_ref (dict);
+
+        while (words[i])
+                len += strlen (words[i++]) + 1;
+
+        cmd = GF_CALLOC (1, len, gf_common_mt_char);
+
+        if (!cmd) {
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; words[i]; i++) {
+                strncat (cmd, words[i], strlen (words[i]));
+                strncat (cmd, " ", strlen (" "));
+        }
+
+        cmd [len - 1] = '\0';
+
+        ret = dict_set_dynstr (dict, "cmd-str", cmd);
+        if (ret)
+                goto out;
+
+        ret = dict_allocate_and_serialize (dict, &(req->dict).dict_val,
+                                           &(req->dict).dict_len);
+
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = cli_cmd_submit (req, frame, prog, procnum, iobref, this,
+                              cbkfn, (xdrproc_t) xdrproc);
+
+out:
+        if (dict)
+                dict_unref (dict);
         return ret;
+
 }
 
 struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
diff --git a/cli/src/cli.h b/cli/src/cli.h
index 4cb235410..0299220ce 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -118,6 +118,7 @@ struct cli_local {
         } get_vol;
 
         dict_t          *dict;
+        const char      **words;
         /* Marker for volume status all */
         gf_boolean_t    all;
 #if (HAVE_LIB_XML)