glusterd: Fixes for detach

Signed-off-by: Vijay Bellur <vijay@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 1215 ()
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1215

8 files changed, 147 insertions, 21 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index df3fd6f071c..3161849e49e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -1653,10 +1653,11 @@ glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
         }
 
         ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND, &event);
+                (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND, &event);
 
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get new event");
+                gf_log ("glusterd", GF_LOG_ERROR,
+                                "Unable to get new event");
                 return ret;
         }
 
@@ -1670,9 +1671,10 @@ glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
         ctx->port = port;
         ctx->req = req;
 
-        event->peerinfo = peerinfo;
         event->ctx = ctx;
 
+        event->peerinfo = peerinfo;
+
         ret = glusterd_friend_sm_inject_event (event);
 
         if (ret) {
@@ -1763,6 +1765,26 @@ glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
         return ret;
 }
 
+int
+glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
+                                int32_t op_errno, char *hostname)
+{
+        gf1_cli_deprobe_rsp    rsp = {0, };
+        int32_t                ret = -1;
+
+        GF_ASSERT (req);
+
+        rsp.op_ret = op_ret;
+        rsp.op_errno = op_errno;
+        rsp.hostname = hostname;
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     gf_xdr_serialize_cli_deprobe_rsp);
+
+        gf_log ("glusterd", GF_LOG_NORMAL, "Responded to CLI, ret: %d",ret);
+
+        return ret;
+}
 int32_t
 glusterd_op_txn_begin ()
 {
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index 30281040ca9..4edc4a1d71d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -239,6 +239,7 @@ glusterd_ac_handle_friend_remove_req (glusterd_friend_sm_event_t *event,
         int                             ret = 0;
         glusterd_peerinfo_t             *peerinfo = NULL;
         glusterd_friend_req_ctx_t       *ev_ctx = NULL;
+        glusterd_friend_sm_event_t      *new_event = NULL;
 
         GF_ASSERT (ctx);
         ev_ctx = ctx;
@@ -250,9 +251,20 @@ glusterd_ac_handle_friend_remove_req (glusterd_friend_sm_event_t *event,
         ret = glusterd_xfer_friend_remove_resp (ev_ctx->req, ev_ctx->hostname,
                                                 ev_ctx->port);
 
-        rpc_clnt_destroy (peerinfo->rpc);
+        //rpc_clnt_destroy (peerinfo->rpc);
         peerinfo->rpc = NULL;
 
+        ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
+                                            &new_event);
+
+        if (ret)
+                goto out;
+
+        new_event->peerinfo = peerinfo;
+
+        ret = glusterd_friend_sm_inject_event (new_event);
+
+out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
@@ -367,8 +379,9 @@ glusterd_sm_t glusterd_state_default [] = {
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_RJT
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_error}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_MAX
 };
 
@@ -382,8 +395,9 @@ glusterd_sm_t  glusterd_state_req_sent [] = {
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
         {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
         {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -396,8 +410,9 @@ glusterd_sm_t  glusterd_state_req_rcvd [] = {
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_error}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -411,7 +426,8 @@ glusterd_sm_t  glusterd_state_befriended [] = {
         {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
         {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -425,7 +441,8 @@ glusterd_sm_t  glusterd_state_req_sent_rcvd [] = {
         {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
         {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -438,8 +455,9 @@ glusterd_sm_t  glusterd_state_rejected [] = {
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -452,8 +470,9 @@ glusterd_sm_t  glusterd_state_req_accepted [] = {
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
         {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -467,6 +486,7 @@ glusterd_sm_t  glusterd_state_unfriend_sent [] = {
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCV_FRIEND_REQ
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_MAX
 };
@@ -570,6 +590,12 @@ glusterd_friend_sm ()
                                 continue;
                         }
 
+                        if ((GD_FRIEND_EVENT_REMOVE_FRIEND == event_type) ||
+                           (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND == event_type)){
+                                GF_FREE (event);
+                                continue;
+                        }
+
                         ret = glusterd_friend_sm_transition_state (peerinfo, state, event_type);
 
                         if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index c49e1f097d2..1ff5baf5913 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -97,6 +97,7 @@ typedef enum glusterd_friend_sm_event_type_ {
         GD_FRIEND_EVENT_LOCAL_RJT,
         GD_FRIEND_EVENT_RCVD_FRIEND_REQ,
         GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
+        GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
         GD_FRIEND_EVENT_REMOVE_FRIEND,
         GD_FRIEND_EVENT_MAX
 } glusterd_friend_sm_event_type_t;
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 6466f9e7f1a..317f65e7975 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1014,6 +1014,58 @@ out:
 }
 
 int32_t
+glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
+{
+        int32_t                         ret = -1;
+        glusterd_conf_t                 *priv = NULL;
+        char                            peerdir[PATH_MAX] = {0,};
+        char                            filepath[PATH_MAX] = {0,};
+        char                            str[512] = {0,};
+        char                            hostname_path[PATH_MAX] = {0,};
+
+
+        if (!peerinfo) {
+                ret = 0;
+                goto out;
+        }
+
+        priv = THIS->private;
+
+        snprintf (peerdir, PATH_MAX, "%s/peers", priv->workdir);
+
+
+        if (uuid_is_null (peerinfo->uuid)) {
+
+                if (peerinfo->hostname) {
+                        snprintf (filepath, PATH_MAX, "%s/%s", peerdir,
+                                  peerinfo->hostname);
+                } else {
+                       ret = 0;
+                       goto out;
+                }
+        } else {
+                uuid_unparse (peerinfo->uuid, str);
+
+                snprintf (filepath, PATH_MAX, "%s/%s", peerdir, str);
+                snprintf (hostname_path, PATH_MAX, "%s/%s",
+                          peerdir, peerinfo->hostname);
+
+                ret = unlink (hostname_path);
+
+                if (!ret)
+                        goto out;
+        }
+
+        ret = unlink (filepath);
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+
+int32_t
 glusterd_store_update_peerinfo (glusterd_peerinfo_t *peerinfo)
 {
         int32_t                         ret = -1;
@@ -1027,7 +1079,11 @@ glusterd_store_update_peerinfo (glusterd_peerinfo_t *peerinfo)
         int                             i = 0;
         char                            hostname_path[PATH_MAX] = {0,};
 
-        GF_ASSERT (peerinfo);
+
+        if (!peerinfo) {
+                ret = 0;
+                goto out;
+        }
 
         priv = THIS->private;
 
@@ -1189,6 +1245,8 @@ glusterd_store_retrieve_peers (xlator_t *this)
         }
 
 out:
+        if (dir)
+                closedir (dir);
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index b9b93d9c91f..9d7a6436e3f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -93,5 +93,8 @@ int32_t
 glusterd_store_update_peerinfo (glusterd_peerinfo_t *peerinfo);
 
 int32_t
+glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo);
+
+int32_t
 glusterd_restore ();
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index e3337c7d985..c17582341c0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -39,6 +39,7 @@
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-store.h"
 
 #include <sys/resource.h>
 #include <inttypes.h>
@@ -858,6 +859,12 @@ glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo)
         if (!peerinfo)
                 goto out;
 
+        ret = glusterd_store_delete_peerinfo (peerinfo);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Deleting peer info failed");
+        }
+
         list_del_init (&peerinfo->uuid_list);
         list_for_each_entry_safe (name, tmp, &peerinfo->hostnames,
                                   hostname_list) {
@@ -867,6 +874,7 @@ glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo)
 
         list_del_init (&peerinfo->hostnames);
         GF_FREE (peerinfo);
+        peerinfo = NULL;
 
         ret = 0;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 33850744dd3..2f0937a326b 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -292,4 +292,8 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req);
 
 int32_t
 glusterd_remove_brick (rpcsvc_request_t *req, dict_t *dict);
+
+int
+glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
+                                int32_t op_errno, char *hostname);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
index 59933e8ac7d..8c84b8d4803 100644
--- a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
+++ b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
@@ -241,7 +241,7 @@ glusterd3_1_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                goto respond;
+                goto inject;
         }
 
         ret = gd_xdr_to_mgmt_friend_rsp (*iov, &rsp);
@@ -263,6 +263,7 @@ glusterd3_1_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
         if (op_ret)
                 goto respond;
 
+inject:
         ret = glusterd_friend_find (rsp.uuid, rsp.hostname, &peerinfo);
 
         if (ret) {
@@ -289,11 +290,12 @@ glusterd3_1_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
         glusterd_friend_sm ();
         glusterd_op_sm ();
 
-        return ret;
+        op_ret = 0;
+
 
 respond:
-        ret = glusterd_xfer_cli_probe_resp (ctx->req, op_ret, op_errno,
-                                            ctx->hostname, ctx->port);
+        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, op_ret, op_errno,
+                                              ctx->hostname);
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
@@ -699,6 +701,8 @@ glusterd3_1_friend_remove (call_frame_t *frame, xlator_t *this,
                                        GD_MGMT_FRIEND_REMOVE,
                                        NULL, gd_xdr_from_mgmt_friend_req,
                                        this, glusterd3_1_friend_remove_cbk);
+        //Override ret
+        ret = 0;
 
 out:
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);