cli,mgmt/glusterd: fsm log implementation

Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 1966 (Unnecessarily verbose logs at the default log level)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1966

11 files changed, 534 insertions, 155 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 930c3a45755..20d2efa0324 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -86,6 +86,8 @@ glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
         if (ret) {
                 ret = glusterd_xfer_friend_add_resp (req, rhost, port, -1,
                                                      GF_PROBE_UNKNOWN_PEER);
+                if (friend_req->vols.vols_val)
+                        free (friend_req->vols.vols_val);
                 goto out;
         }
 
@@ -164,7 +166,6 @@ out:
         return ret;
 }
 
-
 static int
 glusterd_handle_unfriend_req (rpcsvc_request_t *req, uuid_t  uuid,
                               char *hostname, int port)
@@ -1879,6 +1880,85 @@ out:
 }
 
 int
+glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
+                            char *op_errstr, dict_t *dict)
+{
+
+        int                             ret = -1;
+        gf1_cli_fsm_log_rsp             rsp = {0};
+
+        GF_ASSERT (req);
+        GF_ASSERT (op_errstr);
+
+        rsp.op_ret = op_ret;
+        rsp.op_errstr = op_errstr;
+        if (rsp.op_ret == 0)
+                ret = dict_allocate_and_serialize (dict, &rsp.fsm_log.fsm_log_val,
+                                                (size_t *)&rsp.fsm_log.fsm_log_len);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     gf_xdr_from_cli_fsm_log_rsp);
+        if (rsp.fsm_log.fsm_log_val)
+                GF_FREE (rsp.fsm_log.fsm_log_val);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Responded, ret: %d", ret);
+
+        return 0;
+}
+
+int
+glusterd_handle_fsm_log (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf1_cli_fsm_log_req             cli_req = {0,};
+        dict_t                          *dict = NULL;
+        glusterd_sm_tr_log_t            *log = NULL;
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *conf = NULL;
+        char                            msg[2048] = {0};
+        glusterd_peerinfo_t             *peerinfo = NULL;
+
+        GF_ASSERT (req);
+
+        if (!gf_xdr_to_cli_fsm_log_req (req->msg[0], &cli_req)) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                snprintf (msg, sizeof (msg), "Garbage request");
+                goto out;
+        }
+
+        if (strcmp ("", cli_req.name) == 0) {
+                this = THIS;
+                conf = this->private;
+                log = &conf->op_sm_log;
+        } else {
+                ret = glusterd_friend_find_by_hostname (cli_req.name,
+                                                        &peerinfo);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "%s is not a peer",
+                                  cli_req.name);
+                        goto out;
+                }
+                log = &peerinfo->sm_log;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_sm_tr_log_add_to_dict (dict, log);
+out:
+        (void)glusterd_fsm_log_send_resp (req, ret, msg, dict);
+        if (cli_req.name)
+                free (cli_req.name);//malloced by xdr
+        if (dict)
+                dict_unref (dict);
+        return 0;//send 0 to avoid double reply
+}
+
+int
 glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
 {
 
@@ -2335,147 +2415,157 @@ out:
 }
 
 int
-glusterd_friend_add (const char *hoststr, int port,
-                     glusterd_friend_sm_state_t state,
-                     uuid_t *uuid,
-                     struct rpc_clnt    *rpc,
-                     glusterd_peerinfo_t **friend,
-                     gf_boolean_t restore,
-                     glusterd_peerctx_args_t *args)
+glusterd_friend_rpc_create (struct rpc_clnt **rpc,
+                            const char *hoststr, int port,
+                            glusterd_peerctx_t *peerctx)
 {
-        int                     ret = 0;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_peerinfo_t     *peerinfo = NULL;
+        struct rpc_clnt         *new_rpc = NULL;
         dict_t                  *options = NULL;
         struct rpc_clnt_config  rpc_cfg = {0,};
-        glusterd_peer_hostname_t *name = NULL;
+        int                     ret = -1;
         char                    *hostname = NULL;
-        glusterd_peerctx_t     *peerctx = NULL;
         int32_t                 intvl = 0;
+        xlator_t                *this = NULL;
 
-        priv = THIS->private;
+        GF_ASSERT (hoststr);
+        this = THIS;
+        GF_ASSERT (this);
 
-        peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
-        if (!peerctx) {
-                ret = -1;
+        options = dict_new ();
+        if (!options)
                 goto out;
-        }
-        peerinfo = GF_CALLOC (1, sizeof (*peerinfo), gf_gld_mt_peerinfo_t);
 
-        if (!peerinfo) {
-                ret = -1;
-                goto out;
+        ret = dict_get_int32 (this->options,
+                              "transport.socket.keepalive-interval",
+                              &intvl);
+        if (!ret) {
+                ret = dict_set_int32 (options,
+                        "transport.socket.keepalive-interval", intvl);
+                if (ret)
+                        goto out;
         }
 
-        if (args)
-                peerctx->args = *args;
-        peerctx->peerinfo = peerinfo;
-        if (friend)
-                *friend = peerinfo;
-
-        INIT_LIST_HEAD (&peerinfo->hostnames);
-        peerinfo->state.state = state;
-        if (hoststr) {
-                ret =  glusterd_peer_hostname_new ((char *)hoststr, &name);
+        ret = dict_get_int32 (this->options,
+                              "transport.socket.keepalive-time",
+                              &intvl);
+        if (!ret) {
+                ret = dict_set_int32 (options,
+                        "transport.socket.keepalive-time", intvl);
                 if (ret)
                         goto out;
-                list_add_tail (&peerinfo->hostnames, &name->hostname_list);
-                rpc_cfg.remote_host = (char *)hoststr;
-                peerinfo->hostname = gf_strdup (hoststr);
         }
-        INIT_LIST_HEAD (&peerinfo->uuid_list);
-
-        list_add_tail (&peerinfo->uuid_list, &priv->peers);
 
-        if (uuid) {
-                uuid_copy (peerinfo->uuid, *uuid);
+        hostname = gf_strdup((char*)hoststr);
+        if (!hostname) {
+                ret = -1;
+                goto out;
         }
 
+        ret = dict_set_dynstr (options, "remote-host", hostname);
+        if (ret)
+                goto out;
 
-        if (hoststr) {
-                options = dict_new ();
-                if (!options) {
-                        ret = -1;
-                        goto out;
-                }
+        if (!port)
+                port = GLUSTERD_DEFAULT_PORT;
 
-                ret = dict_get_int32 (THIS->options,
-                                      "transport.socket.keepalive-interval",
-                                      &intvl);
-                if (!ret) {
-                        ret = dict_set_int32 (options,
-                                "transport.socket.keepalive-interval", intvl);
-                        if (ret)
-                                goto out;
-                }
+        rpc_cfg.remote_host = (char *)hoststr;
+        rpc_cfg.remote_port = port;
 
-                ret = dict_get_int32 (THIS->options,
-                                      "transport.socket.keepalive-time",
-                                      &intvl);
-                if (!ret) {
-                        ret = dict_set_int32 (options,
-                                "transport.socket.keepalive-time", intvl);
-                        if (ret)
-                                goto out;
-                }
+        ret = dict_set_int32 (options, "remote-port", port);
+        if (ret)
+                goto out;
 
-                hostname = gf_strdup((char*)hoststr);
-                if (!hostname) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_str (options, "transport.address-family", "inet");
+        if (ret)
+                goto out;
 
-                ret = dict_set_dynstr (options, "remote-host", hostname);
-                if (ret)
-                        goto out;
+        new_rpc = rpc_clnt_init (&rpc_cfg, options, this->ctx, this->name);
 
+        if (!new_rpc) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "new_rpc init failed for peer: %s!", hoststr);
+                ret = -1;
+                goto out;
+        }
 
-                if (!port)
-                        port = GLUSTERD_DEFAULT_PORT;
+        ret = rpc_clnt_register_notify (new_rpc, glusterd_rpc_notify,
+                                        peerctx);
+        *rpc = new_rpc;
+out:
+        if (ret) {
+                if (new_rpc) {
+                        (void) rpc_clnt_unref (new_rpc);
+                }
+                if (options)
+                        dict_unref (options);
+                *rpc = NULL;
+        }
 
-                rpc_cfg.remote_port = port;
+        gf_log ("", GF_LOG_DEBUG, "returning %d");
+        return ret;
+}
 
-                ret = dict_set_int32 (options, "remote-port", port);
-                if (ret)
-                        goto out;
+int
+glusterd_friend_add (const char *hoststr, int port,
+                     glusterd_friend_sm_state_t state,
+                     uuid_t *uuid,
+                     struct rpc_clnt    *rpc,
+                     glusterd_peerinfo_t **friend,
+                     gf_boolean_t restore,
+                     glusterd_peerctx_args_t *args)
+{
+        int                     ret = 0;
+        glusterd_conf_t         *conf = NULL;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        glusterd_peerctx_t     *peerctx = NULL;
+        gf_boolean_t            is_allocated = _gf_false;
 
-                ret = dict_set_str (options, "transport.address-family", "inet");
-                if (ret)
-                        goto out;
+        conf = THIS->private;
+        GF_ASSERT (conf)
 
-                rpc = rpc_clnt_init (&rpc_cfg, options, THIS->ctx, THIS->name);
+        peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
+        if (!peerctx) {
+                ret = -1;
+                goto out;
+        }
 
-                if (!rpc) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "rpc init failed for peer: %s!", hoststr);
-                        ret = -1;
-                        goto out;
-                }
+        if (args)
+                peerctx->args = *args;
 
-                ret = rpc_clnt_register_notify (rpc, glusterd_rpc_notify,
-                                                peerctx);
+        ret = glusterd_peerinfo_new (&peerinfo, state, uuid, hoststr);
+        if (ret)
+                goto out;
+        peerctx->peerinfo = peerinfo;
+        if (friend)
+                *friend = peerinfo;
 
+        if (hoststr) {
+                if (!rpc) {
+                        ret = glusterd_friend_rpc_create (&rpc, hoststr, port,
+                                                          peerctx);
+                        if (ret)
+                                goto out;
+                        is_allocated = _gf_true;
+                }
                 peerinfo->rpc = rpc;
-
         }
 
         if (!restore)
                 ret = glusterd_store_update_peerinfo (peerinfo);
 
+        list_add_tail (&peerinfo->uuid_list, &conf->peers);
 
 out:
         if (ret) {
                 if (peerctx)
                         GF_FREE (peerctx);
-                if (rpc) {
+                if (is_allocated && rpc) {
                         (void) rpc_clnt_unref (rpc);
                 }
                 if (peerinfo) {
                         peerinfo->rpc = NULL;
                         (void) glusterd_friend_cleanup (peerinfo);
                 }
-                if (options)
-                        dict_unref (options);
         }
 
         gf_log ("glusterd", GF_LOG_NORMAL, "connect returned %d", ret);
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index 5ee9375dd16..5a4da805671 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -59,7 +59,8 @@ enum gf_gld_mem_types_ {
         gf_gld_mt_log_locate_ctx_t              = gf_common_mt_end + 33,
         gf_gld_mt_log_rotate_ctx_t              = gf_common_mt_end + 34,
         gf_gld_mt_peerctx_t                     = gf_common_mt_end + 35,
-        gf_gld_mt_end                           = gf_common_mt_end + 36
+        gf_gld_mt_sm_tr_log_t                = gf_common_mt_end + 36,
+        gf_gld_mt_end                           = gf_common_mt_end + 37
 };
 #endif
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 93eeb38d529..bc9cb330bdc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -87,7 +87,7 @@ static char *glusterd_op_sm_event_names[] = {
 };
 
 char*
-glusterd_op_sm_state_name_get (glusterd_op_sm_state_t state)
+glusterd_op_sm_state_name_get (int state)
 {
         if (state < 0 || state >= GD_OP_STATE_MAX)
                 return glusterd_op_sm_state_names[GD_OP_STATE_MAX];
@@ -95,7 +95,7 @@ glusterd_op_sm_state_name_get (glusterd_op_sm_state_t state)
 }
 
 char*
-glusterd_op_sm_event_name_get (glusterd_op_sm_event_type_t event)
+glusterd_op_sm_event_name_get (int event)
 {
         if (event < 0 || event >= GD_OP_EVENT_MAX)
                 return glusterd_op_sm_event_names[GD_OP_EVENT_MAX];
@@ -4772,15 +4772,19 @@ glusterd_op_sm_transition_state (glusterd_op_info_t *opinfo,
                                  glusterd_op_sm_t *state,
                                  glusterd_op_sm_event_type_t event_type)
 {
+        glusterd_conf_t         *conf = NULL;
 
         GF_ASSERT (state);
         GF_ASSERT (opinfo);
 
-        gf_log ("", GF_LOG_NORMAL, "Transitioning from '%s' to '%s' due to "
-                "event '%s'",
-                glusterd_op_sm_state_name_get (opinfo->state.state),
-                glusterd_op_sm_state_name_get (state[event_type].next_state),
-                glusterd_op_sm_event_name_get (event_type));
+        conf = THIS->private;
+        GF_ASSERT (conf);
+
+        (void) glusterd_sm_tr_log_transition_add (&conf->op_sm_log,
+                                           opinfo->state.state,
+                                           state[event_type].next_state,
+                                           event_type);
+
         opinfo->state.state = state[event_type].next_state;
         return 0;
 }
@@ -5100,7 +5104,7 @@ glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
 
         event->ctx = ctx;
 
-        gf_log ("glusterd", GF_LOG_NORMAL, "Enqueuing event: '%s'",
+        gf_log ("glusterd", GF_LOG_DEBUG, "Enqueuing event: '%s'",
                 glusterd_op_sm_event_name_get (event->event));
         list_add_tail (&event->list, &gd_op_sm_queue);
 
@@ -5148,7 +5152,7 @@ glusterd_op_sm ()
 
                         list_del_init (&event->list);
                         event_type = event->event;
-                        gf_log ("", GF_LOG_NORMAL, "Dequeued event of type: '%s'",
+                        gf_log ("", GF_LOG_DEBUG, "Dequeued event of type: '%s'",
                                 glusterd_op_sm_event_name_get(event_type));
 
                         state = glusterd_op_state_table[opinfo.state.state];
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index f6588051298..27314781b2b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -228,4 +228,10 @@ void
 glusterd_do_replace_brick (void *data);
 int
 glusterd_options_reset (glusterd_volinfo_t *volinfo);
+
+char*
+glusterd_op_sm_state_name_get (int state);
+
+char*
+glusterd_op_sm_event_name_get (int event);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index d862a697d48..d572ee70c2b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -75,7 +75,7 @@ static char *glusterd_friend_sm_event_names[] = {
 };
 
 char*
-glusterd_friend_sm_state_name_get (glusterd_friend_sm_state_t state)
+glusterd_friend_sm_state_name_get (int state)
 {
         if (state < 0 || state >= GD_FRIEND_STATE_MAX)
                 return glusterd_friend_sm_state_names[GD_FRIEND_STATE_MAX];
@@ -83,7 +83,7 @@ glusterd_friend_sm_state_name_get (glusterd_friend_sm_state_t state)
 }
 
 char*
-glusterd_friend_sm_event_name_get (glusterd_friend_sm_event_type_t event)
+glusterd_friend_sm_event_name_get (int event)
 {
         if (event < 0 || event >= GD_FRIEND_EVENT_MAX)
                 return glusterd_friend_sm_event_names[GD_FRIEND_EVENT_MAX];
@@ -538,13 +538,10 @@ glusterd_friend_sm_transition_state (glusterd_peerinfo_t *peerinfo,
         GF_ASSERT (state);
         GF_ASSERT (peerinfo);
 
-        //peerinfo->state.state = state;
-
-        gf_log ("", GF_LOG_NORMAL, "Transitioning from '%s' to '%s' due to "
-                "event '%s'",
-                glusterd_friend_sm_state_name_get (peerinfo->state.state),
-                glusterd_friend_sm_state_name_get (state[event_type].next_state),
-                glusterd_friend_sm_event_name_get (event_type));
+        (void) glusterd_sm_tr_log_transition_add (&peerinfo->sm_log,
+                                           peerinfo->state.state,
+                                           state[event_type].next_state,
+                                           event_type);
 
         peerinfo->state.state = state[event_type].next_state;
         return 0;
@@ -717,7 +714,7 @@ int
 glusterd_friend_sm_inject_event (glusterd_friend_sm_event_t *event)
 {
         GF_ASSERT (event);
-        gf_log ("glusterd", GF_LOG_NORMAL, "Enqueuing event: '%s'",
+        gf_log ("glusterd", GF_LOG_DEBUG, "Enqueuing event: '%s'",
                 glusterd_friend_sm_event_name_get (event->event));
         list_add_tail (&event->list, &gd_friend_sm_queue);
 
@@ -767,11 +764,14 @@ glusterd_friend_sm ()
                         if (!peerinfo) {
                                 gf_log ("glusterd", GF_LOG_CRITICAL, "Received"
                                         " event %s with empty peer info",
-                                glusterd_friend_sm_event_name_get(event_type));
+                                glusterd_friend_sm_event_name_get (event_type));
 
                                 GF_FREE (event);
                                 continue;
                         }
+                        gf_log ("", GF_LOG_DEBUG, "Dequeued event of type: '%s'",
+                                glusterd_friend_sm_event_name_get (event_type));
+
 
                         state = glusterd_friend_state_table[peerinfo->state.state];
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index ebe6cb379c0..41ec2694317 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -68,6 +68,22 @@ typedef struct glusterd_peer_hostname_ {
         struct list_head        hostname_list;
 }glusterd_peer_hostname_t;
 
+typedef struct glusterd_sm_transition_ {
+        int             old_state;
+        int             event;
+        int             new_state;
+        time_t          time;
+} glusterd_sm_transition_t;
+
+typedef struct glusterd_sm_tr_log_ {
+        glusterd_sm_transition_t    *transitions;
+        size_t                      current;
+        size_t                      size;
+        size_t                      count;
+        char*                       (*state_name_get) (int);
+        char*                       (*event_name_get) (int);
+} glusterd_sm_tr_log_t;
+
 struct glusterd_peerinfo_ {
         uuid_t                          uuid;
         char                            uuid_str[50];
@@ -80,6 +96,7 @@ struct glusterd_peerinfo_ {
         struct rpc_clnt                 *rpc;
         int                             connected;
         glusterd_store_handle_t         *shandle;
+        glusterd_sm_tr_log_t            sm_log;
 };
 
 typedef struct glusterd_peerinfo_ glusterd_peerinfo_t;
@@ -178,7 +195,10 @@ void
 glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx);
 
 char*
-glusterd_friend_sm_state_name_get (glusterd_friend_sm_state_t state);
+glusterd_friend_sm_state_name_get (int state);
+
+char*
+glusterd_friend_sm_event_name_get (int event);
 
 int
 glusterd_broadcast_friend_delete (char *hostname, uuid_t uuid);
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 0634c4b30f9..cffd9dbd458 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -765,7 +765,8 @@ glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo)
                 peerinfo->rpc->mydata = NULL;
                 peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
                 peerinfo->rpc = NULL;
-                GF_FREE (peerctx);
+                if (peerctx)
+                        GF_FREE (peerctx);
         }
         glusterd_peer_destroy (peerinfo);
 
@@ -1052,43 +1053,6 @@ out:
         return ret;
 }
 
-int32_t
-glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo)
-{
-        int32_t                         ret = -1;
-        glusterd_peer_hostname_t        *name = NULL;
-        glusterd_peer_hostname_t        *tmp = NULL;
-
-        if (!peerinfo)
-                goto out;
-
-        ret = glusterd_store_delete_peerinfo (peerinfo);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Deleting peer info failed");
-        }
-
-        list_del_init (&peerinfo->uuid_list);
-        list_for_each_entry_safe (name, tmp, &peerinfo->hostnames,
-                                  hostname_list) {
-                list_del_init (&name->hostname_list);
-                GF_FREE (name->hostname);
-                GF_FREE (name);
-        }
-
-        list_del_init (&peerinfo->hostnames);
-        if (peerinfo->hostname)
-                GF_FREE (peerinfo->hostname);
-        GF_FREE (peerinfo);
-        peerinfo = NULL;
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
 gf_boolean_t
 glusterd_is_cli_op_req (int32_t op)
 {
@@ -2429,3 +2393,259 @@ out:
         gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
+
+int
+glusterd_sm_tr_log_transition_add_to_dict (dict_t *dict,
+                                           glusterd_sm_tr_log_t *log, int i,
+                                           int count)
+{
+        int     ret = -1;
+        char    key[512] = {0};
+	char    timestr[256] = {0,};
+        char    *str = NULL;
+	struct tm   tm = {0};
+
+        GF_ASSERT (dict);
+        GF_ASSERT (log);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "log%d-old-state", count);
+        str = log->state_name_get (log->transitions[i].old_state);
+        ret = dict_set_str (dict, key, str);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "log%d-event", count);
+        str = log->event_name_get (log->transitions[i].event);
+        ret = dict_set_str (dict, key, str);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "log%d-new-state", count);
+        str = log->state_name_get (log->transitions[i].new_state);
+        ret = dict_set_str (dict, key, str);
+        if (ret)
+                goto out;
+
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "log%d-time", count);
+	localtime_r ((const time_t*)&log->transitions[i].time, &tm);
+        memset (timestr, 0, sizeof (timestr));
+	strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", &tm);
+        str = gf_strdup (timestr);
+        ret = dict_set_dynstr (dict, key, str);
+        if (ret)
+                goto out;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_sm_tr_log_add_to_dict (dict_t *dict,
+                                glusterd_sm_tr_log_t *circular_log)
+{
+        int     ret = -1;
+        int     i = 0;
+        int     start = 0;
+        int     end     = 0;
+        int     index = 0;
+        char    key[256] = {0};
+        glusterd_sm_tr_log_t *log = NULL;
+        int     count = 0;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (circular_log);
+
+        log = circular_log;
+        if (!log->count)
+                return 0;
+
+        if (log->count == log->size)
+                start = log->current + 1;
+
+        end = start + log->count;
+        for (i = start; i < end; i++, count++) {
+                index = i % log->count;
+                ret = glusterd_sm_tr_log_transition_add_to_dict (dict, log, index,
+                                                                 count);
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "count");
+        ret = dict_set_int32 (dict, key, log->count);
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_sm_tr_log_init (glusterd_sm_tr_log_t *log,
+                         char * (*state_name_get) (int),
+                         char * (*event_name_get) (int),
+                         size_t  size)
+{
+        glusterd_sm_transition_t *transitions = NULL;
+        int                      ret = -1;
+
+        GF_ASSERT (size > 0);
+        GF_ASSERT (log && state_name_get && event_name_get);
+
+        if (!log || !state_name_get || !event_name_get || (size <= 0))
+                goto out;
+
+        transitions = GF_CALLOC (size, sizeof (*transitions),
+                                 gf_gld_mt_sm_tr_log_t);
+        if (!transitions)
+                goto out;
+
+        log->transitions = transitions;
+        log->size        = size;
+        log->state_name_get = state_name_get;
+        log->event_name_get = event_name_get;
+        ret = 0;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+void
+glusterd_sm_tr_log_delete (glusterd_sm_tr_log_t *log)
+{
+        if (!log)
+                return;
+        if (log->transitions)
+                GF_FREE (log->transitions);
+        return;
+}
+
+int
+glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
+                                   int old_state, int new_state,
+                                   int event)
+{
+        glusterd_sm_transition_t *transitions = NULL;
+        int                      ret = -1;
+        int                      next = 0;
+
+        GF_ASSERT (log);
+        if (!log)
+                goto out;
+
+        transitions = log->transitions;
+        if (!transitions)
+                goto out;
+
+        if (log->count)
+                next = (log->current + 1) % log->size;
+        else
+                next = 0;
+
+        transitions[next].old_state = old_state;
+        transitions[next].new_state = new_state;
+        transitions[next].event     = event;
+        time (&transitions[next].time);
+        log->current = next;
+        if (log->count < log->size)
+                log->count++;
+        ret = 0;
+        gf_log ("glusterd", GF_LOG_DEBUG, "Transitioning from '%s' to '%s' "
+                "due to event '%s'", log->state_name_get (old_state),
+                log->state_name_get (new_state), log->event_name_get (event));
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
+                       glusterd_friend_sm_state_t state,
+                       uuid_t *uuid, const char *hostname)
+{
+        glusterd_peerinfo_t      *new_peer = NULL;
+        glusterd_peer_hostname_t *name = NULL;
+        int                      ret = -1;
+
+        GF_ASSERT (peerinfo);
+        if (!peerinfo)
+                goto out;
+
+        new_peer = GF_CALLOC (1, sizeof (*new_peer), gf_gld_mt_peerinfo_t);
+        if (!new_peer)
+                goto out;
+
+        INIT_LIST_HEAD (&new_peer->hostnames);
+        new_peer->state.state = state;
+        if (hostname) {
+                ret =  glusterd_peer_hostname_new ((char *)hostname, &name);
+                if (ret)
+                        goto out;
+                list_add_tail (&new_peer->hostnames, &name->hostname_list);
+                new_peer->hostname = gf_strdup (hostname);
+        }
+
+        INIT_LIST_HEAD (&new_peer->uuid_list);
+
+        if (uuid) {
+                uuid_copy (new_peer->uuid, *uuid);
+        }
+
+        ret = glusterd_sm_tr_log_init (&new_peer->sm_log,
+                                       glusterd_friend_sm_state_name_get,
+                                       glusterd_friend_sm_event_name_get,
+                                       GLUSTERD_TR_LOG_SIZE);
+        if (ret)
+                goto out;
+
+        *peerinfo = new_peer;
+out:
+        if (ret && new_peer)
+                glusterd_friend_cleanup (new_peer);
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo)
+{
+        int32_t                         ret = -1;
+        glusterd_peer_hostname_t        *name = NULL;
+        glusterd_peer_hostname_t        *tmp = NULL;
+
+        if (!peerinfo)
+                goto out;
+
+        ret = glusterd_store_delete_peerinfo (peerinfo);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Deleting peer info failed");
+        }
+
+        list_del_init (&peerinfo->uuid_list);
+        list_for_each_entry_safe (name, tmp, &peerinfo->hostnames,
+                                  hostname_list) {
+                list_del_init (&name->hostname_list);
+                GF_FREE (name->hostname);
+                GF_FREE (name);
+        }
+
+        list_del_init (&peerinfo->hostnames);
+        if (peerinfo->hostname)
+                GF_FREE (peerinfo->hostname);
+        glusterd_sm_tr_log_delete (&peerinfo->sm_log);
+        GF_FREE (peerinfo);
+        peerinfo = NULL;
+
+        ret = 0;
+
+out:
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 5797c35cfdf..8d4fac48942 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -230,4 +230,23 @@ glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
 int
 glusterd_brick_create_path (char *host, char *path, mode_t mode,
                             char **op_errstr);
+int
+glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
+                                           int old_state, int new_state,
+                                           int event);
+int
+glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
+                       glusterd_friend_sm_state_t state,
+                       uuid_t *uuid, const char *hostname);
+int
+glusterd_sm_tr_log_init (glusterd_sm_tr_log_t *log,
+                         char * (*state_name_get) (int),
+                         char * (*event_name_get) (int),
+                         size_t  size);
+void
+glusterd_sm_tr_log_delete (glusterd_sm_tr_log_t *log);
+
+int
+glusterd_sm_tr_log_add_to_dict (dict_t *dict,
+                                glusterd_sm_tr_log_t *circular_log);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 9d9820e8450..78328b98007 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -43,6 +43,7 @@
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-store.h"
+#include "glusterd-utils.h"
 
 static uuid_t glusterd_uuid;
 extern struct rpcsvc_program glusterd1_mop_prog;
@@ -386,6 +387,12 @@ init (xlator_t *this)
         strncpy (conf->workdir, dirname, PATH_MAX);
 
         INIT_LIST_HEAD (&conf->xprt_list);
+        ret = glusterd_sm_tr_log_init (&conf->op_sm_log,
+                                       glusterd_op_sm_state_name_get,
+                                       glusterd_op_sm_event_name_get,
+                                       GLUSTERD_TR_LOG_SIZE);
+        if (ret)
+                goto out;
 
         this->private = conf;
         //this->ctx->top = this;
@@ -437,6 +444,7 @@ fini (xlator_t *this)
                 FREE (conf->pmap);
         if (conf->handle)
                 glusterd_store_handle_destroy (conf->handle);
+        glusterd_sm_tr_log_delete (&conf->op_sm_log);
         GF_FREE (conf);
         this->private = NULL;
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index dfa8b73e56b..11774a3e9d0 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -49,6 +49,8 @@
 
 #define GLUSTERD_MAX_VOLUME_NAME        1000
 #define DEFAULT_LOG_FILE_DIRECTORY      DATADIR "/log/glusterfs"
+#define GLUSTERD_TR_LOG_SIZE            50
+
 
 typedef enum glusterd_op_ {
         GD_OP_NONE = 0,
@@ -96,6 +98,7 @@ typedef struct {
         struct list_head  xprt_list;
         glusterd_store_handle_t *handle;
         gf_timer_t *timer;
+        glusterd_sm_tr_log_t op_sm_log;
 } glusterd_conf_t;
 
 typedef enum gf_brick_status {
@@ -440,6 +443,9 @@ int
 glusterd_handle_reset_volume (rpcsvc_request_t *req);
 
 int
+glusterd_handle_fsm_log (rpcsvc_request_t *req);
+
+int
 glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
                                 int32_t op_errno, char *hostname);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
index 8e9c0806fcb..3004954ce4d 100644
--- a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
+++ b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
@@ -1483,6 +1483,10 @@ glusterd_handle_rpc_msg (rpcsvc_request_t *req)
                         ret = glusterd_handle_reset_volume (req);
                         break;
 
+                case GD_MGMT_CLI_FSM_LOG:
+                        ret = glusterd_handle_fsm_log (req);
+                        break;
+
                 default:
 			gf_log("", GF_LOG_ERROR, "Recieved Invalid procnum:%d",
 			       req->procnum);
@@ -1538,7 +1542,8 @@ rpcsvc_actor_t glusterd1_mgmt_actors[] = {
         [GD_MGMT_CLI_LOG_ROTATE] = { "LOG FILENAME", GD_MGMT_CLI_LOG_ROTATE, glusterd_handle_rpc_msg, NULL, NULL},
         [GD_MGMT_CLI_SET_VOLUME] = { "SET_VOLUME", GD_MGMT_CLI_SET_VOLUME, glusterd_handle_rpc_msg, NULL, NULL},
         [GD_MGMT_CLI_SYNC_VOLUME] = { "SYNC_VOLUME", GD_MGMT_CLI_SYNC_VOLUME, glusterd_handle_rpc_msg, NULL, NULL},
-        [GD_MGMT_CLI_RESET_VOLUME] = { "RESET_VOLUME", GD_MGMT_CLI_RESET_VOLUME, glusterd_handle_rpc_msg, NULL, NULL}
+        [GD_MGMT_CLI_RESET_VOLUME] = { "RESET_VOLUME", GD_MGMT_CLI_RESET_VOLUME, glusterd_handle_rpc_msg, NULL, NULL},
+        [GD_MGMT_CLI_FSM_LOG] = { "FSM_LOG", GD_MGMT_CLI_FSM_LOG, glusterd_handle_rpc_msg, NULL, NULL}
 };
 
 /*rpcsvc_actor_t glusterd1_mgmt_actors[] = {