mgmt/glusterd: handle reqs from unknown peers for friend sm

Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1838 (handle peer detach gracefully in case of lost frames)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1838

6 files changed, 126 insertions, 60 deletions

diff --git a/cli/src/cli3_1-cops.c b/cli/src/cli3_1-cops.c
index 5b5d5425f..ee66b165f 100644
--- a/cli/src/cli3_1-cops.c
+++ b/cli/src/cli3_1-cops.c
@@ -121,6 +121,17 @@ gf_cli3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                                          "with existing volumes in the "
                                          "cluster", rsp.hostname);
                                 break;
+                        case GF_PROBE_UNKNOWN_PEER:
+                                cli_out ("%s responded with 'unknown peer' error, "
+                                         "this could happen if %s doesn't have"
+                                         " localhost in its peer database",
+                                         rsp.hostname, rsp.hostname);
+                                break;
+                        case GF_PROBE_ADD_FAILED:
+                                cli_out ("Failed to add peer information "
+                                         "on %s" , rsp.hostname);
+                                break;
+
                         default:
                                 cli_out ("Probe returned with unknown errno %d",
                                         rsp.op_errno);
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index 40283bf42..bfd043e31 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -161,7 +161,9 @@ enum gf_probe_resp {
 	GF_PROBE_LOCALHOST,
 	GF_PROBE_FRIEND,
         GF_PROBE_ANOTHER_CLUSTER,
-        GF_PROBE_VOLUME_CONFLICT
+        GF_PROBE_VOLUME_CONFLICT,
+        GF_PROBE_UNKNOWN_PEER,
+        GF_PROBE_ADD_FAILED
 };
 
 enum gf_deprobe_resp {
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 8cf7f7444..a53ac4da9 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -328,7 +328,7 @@ saved_frames_unwind (struct saved_frames *saved_frames)
                 if (!trav->rpcreq || !trav->rpcreq->prog)
                         continue;
 
-                gf_log ("rpc-clnt", GF_LOG_ERROR,
+                gf_log_callingfn ("rpc-clnt", GF_LOG_ERROR,
                         "forced unwinding frame type(%s) op(%s(%d)) "
                         "called at %s",
                         trav->rpcreq->prog->progname,
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 49281b956..7d5b749cf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -63,16 +63,17 @@
 
 static int
 glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
-                            char *hostname, int port, dict_t *dict)
+                            char *hostname, int port,
+                            gd1_mgmt_friend_req *friend_req)
 {
         int                             ret = -1;
         glusterd_peerinfo_t             *peerinfo = NULL;
         glusterd_friend_sm_event_t      *event = NULL;
         glusterd_friend_req_ctx_t       *ctx = NULL;
-        glusterd_peerctx_args_t         args = {0};
         char                            rhost[UNIX_PATH_MAX + 1] = {0};
         uuid_t                          friend_uuid = {0};
         char                            uuid_str[50] = {0,};
+        dict_t                          *dict = NULL;
 
         uuid_unparse (uuid, uuid_str);
         uuid_parse (uuid_str, friend_uuid);
@@ -83,15 +84,9 @@ glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
         ret = glusterd_friend_find (uuid, rhost, &peerinfo);
 
         if (ret) {
-                gf_log ("glusterd", GF_LOG_NORMAL,
-                        " for host: %s (%d)", rhost, port);
-                args.mode = GD_MODE_SWITCH_ON;
-                args.req  = NULL;
-                ret = glusterd_friend_add (rhost, port,
-                                           GD_FRIEND_STATE_DEFAULT,
-                                           &friend_uuid, NULL, &peerinfo, 0,
-                                           &args);
-
+                ret = glusterd_xfer_friend_add_resp (req, rhost, port, -1,
+                                                     GF_PROBE_UNKNOWN_PEER);
+                goto out;
         }
 
         ret = glusterd_friend_sm_new_event
@@ -116,8 +111,23 @@ glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
         if (hostname)
                 ctx->hostname = gf_strdup (hostname);
         ctx->req = req;
-        ctx->vols = dict;
 
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (friend_req->vols.vols_val,
+                                friend_req->vols.vols_len,
+                                &dict);
+
+        if (ret)
+                goto out;
+        else
+                dict->extra_stdfree = friend_req->vols.vols_val;
+
+        ctx->vols = dict;
         event->ctx = ctx;
 
         ret = glusterd_friend_sm_inject_event (event);
@@ -134,14 +144,23 @@ out:
         if (0 != ret) {
                 if (ctx && ctx->hostname)
                         GF_FREE (ctx->hostname);
-                if (ctx && ctx->vols)
-                        dict_destroy (ctx->vols);
                 if (ctx)
                         GF_FREE (ctx);
+                if (dict) {
+                        if ((!dict->extra_stdfree) &&
+                            friend_req->vols.vols_val)
+                                free (friend_req->vols.vols_val);
+                        dict_unref (dict);
+                } else {
+                    if (friend_req->vols.vols_val)
+                        free (friend_req->vols.vols_val);
+                }
+                if (event)
+                        GF_FREE (event);
+        } else {
+                if (peerinfo && (0 == peerinfo->connected))
+                        ret = GLUSTERD_CONNECTION_AWAITED;
         }
-
-        if (0 == peerinfo->connected)
-                ret = GLUSTERD_CONNECTION_AWAITED;
         return ret;
 }
 
@@ -161,9 +180,12 @@ glusterd_handle_unfriend_req (rpcsvc_request_t *req, uuid_t  uuid,
         ret = glusterd_friend_find (uuid, hostname, &peerinfo);
 
         if (ret) {
-                gf_log ("glusterd", GF_LOG_NORMAL,
-                         "Unable to find peer");
-
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Received remove-friend from unknown peer %s",
+                        hostname);
+                ret = glusterd_xfer_friend_remove_resp (req, hostname,
+                                                        port);
+                goto out;
         }
 
         ret = glusterd_friend_sm_new_event
@@ -1919,6 +1941,8 @@ glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
 
         gf_log ("glusterd", GF_LOG_NORMAL,
                 "Responded to stage, ret: %d", ret);
+        if (rsp.dict.dict_val)
+                GF_FREE (rsp.dict.dict_val);
 
         return ret;
 }
@@ -1969,7 +1993,6 @@ glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
         int32_t                 ret = -1;
         gd1_mgmt_friend_req     friend_req = {{0},};
         char                    str[50] = {0,};
-        dict_t                  *dict = NULL;
 
         GF_ASSERT (req);
         if (!gd_xdr_to_mgmt_friend_req (req->msg[0], &friend_req)) {
@@ -1981,31 +2004,11 @@ glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
 
         gf_log ("glusterd", GF_LOG_NORMAL,
                 "Received probe from uuid: %s", str);
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (friend_req.vols.vols_val,
-                                friend_req.vols.vols_len,
-                                &dict);
-
-        if (ret)
-                goto out;
-        else
-                dict->extra_stdfree = friend_req.vols.vols_val;
-
         ret = glusterd_handle_friend_req (req, friend_req.uuid,
                                           friend_req.hostname, friend_req.port,
-                                          dict);
+                                          &friend_req);
 
 out:
-        if (GLUSTERD_CONNECTION_AWAITED != ret) {
-                if (ret && dict)
-                        dict_unref (dict);
-        }
         if (friend_req.hostname)
                 free (friend_req.hostname);//malloced by xdr
 
@@ -2018,6 +2021,7 @@ glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
         int32_t                 ret = -1;
         gd1_mgmt_friend_req     friend_req = {{0},};
         char                    str[50];
+        char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
 
         GF_ASSERT (req);
         if (!gd_xdr_to_mgmt_friend_req (req->msg[0], &friend_req)) {
@@ -2030,8 +2034,14 @@ glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
         gf_log ("glusterd", GF_LOG_NORMAL,
                 "Received unfriend from uuid: %s", str);
 
+        ret = glusterd_remote_hostname_get (req, remote_hostname,
+                                            sizeof (remote_hostname));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get the remote hostname");
+                goto out;
+        }
         ret = glusterd_handle_unfriend_req (req, friend_req.uuid,
-                                            friend_req.hostname, friend_req.port);
+                                            remote_hostname, friend_req.port);
 
 out:
         if (friend_req.hostname)
@@ -2097,6 +2107,12 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
         }
         uuid_unparse (friend_req.uuid, str);
 
+        ret = glusterd_friend_find (friend_req.uuid, NULL, &tmp);
+        if (ret) {
+                gf_log ("", GF_LOG_CRITICAL, "Received friend update request "
+                        "from unknown peer %s", str);
+                goto out;
+        }
         gf_log ("glusterd", GF_LOG_NORMAL,
                 "Received friend update from uuid: %s", str);
 
@@ -2169,21 +2185,30 @@ out:
         uuid_copy (rsp.uuid, priv->uuid);
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      gd_xdr_serialize_mgmt_friend_update_rsp);
-        if (dict)
+        if (dict) {
+                if (!dict->extra_stdfree && friend_req.friends.friends_val)
+                        free (friend_req.friends.friends_val);//malloced by xdr
                 dict_unref (dict);
+        } else {
+                if (friend_req.friends.friends_val)
+                        free (friend_req.friends.friends_val);//malloced by xdr
+        }
+
         return ret;
 }
 
 int
 glusterd_handle_probe_query (rpcsvc_request_t *req)
 {
-        int32_t             ret = -1;
-        char                str[50];
-        xlator_t            *this = NULL;
-        glusterd_conf_t     *conf = NULL;
-        gd1_mgmt_probe_req  probe_req = {{0},};
-        gd1_mgmt_probe_rsp  rsp = {{0},};
+        int32_t                         ret = -1;
+        char                            str[50];
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *conf = NULL;
+        gd1_mgmt_probe_req              probe_req = {{0},};
+        gd1_mgmt_probe_rsp              rsp = {{0},};
         glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_peerctx_args_t         args = {0};
+        int                             port = 0;
         char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
 
         GF_ASSERT (req);
@@ -2199,6 +2224,10 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
 
         conf = this->private;
         uuid_unparse (probe_req.uuid, str);
+        if (probe_req.port)
+                port = probe_req.port;
+        else
+                port = GF_DEFAULT_BASE_PORT;
 
         gf_log ("glusterd", GF_LOG_NORMAL,
                 "Received probe from uuid: %s", str);
@@ -2206,13 +2235,25 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
         ret = glusterd_remote_hostname_get (req, remote_hostname,
                                             sizeof (remote_hostname));
         if (ret) {
-                GF_ASSERT (0);
+                gf_log ("", GF_LOG_ERROR, "Unable to get the remote hostname");
                 goto out;
         }
         ret = glusterd_friend_find (probe_req.uuid, remote_hostname, &peerinfo);
         if ((ret != 0 ) && (!list_empty (&conf->peers))) {
                 rsp.op_ret = -1;
                 rsp.op_errno = GF_PROBE_ANOTHER_CLUSTER;
+        } else if (ret) {
+                gf_log ("glusterd", GF_LOG_NORMAL, "Unable to find peerinfo"
+                        " for host: %s (%d)", remote_hostname, port);
+                args.mode = GD_MODE_SWITCH_ON;
+                ret = glusterd_friend_add (remote_hostname, port,
+                                           GD_FRIEND_STATE_DEFAULT,
+                                           NULL, NULL, &peerinfo, 0, &args);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to add peer %s",
+                                remote_hostname);
+                        rsp.op_errno = GF_PROBE_ADD_FAILED;
+                }
         }
 
         uuid_copy (rsp.uuid, conf->uuid);
@@ -2419,7 +2460,7 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
                                            NULL, NULL, &peerinfo, 0, &args);
         }
 
-        if (!peerinfo->connected) {
+        if ((!ret) && (!peerinfo->connected)) {
                 return  GLUSTERD_CONNECTION_AWAITED;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index 6061d0276..385e62a40 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -765,7 +765,12 @@ glusterd_friend_sm ()
                         event_type = event->event;
                         peerinfo = event->peerinfo;
                         if (!peerinfo) {
-                                GF_ASSERT (0);
+                                gf_log ("glusterd", GF_LOG_CRITICAL, "Received"
+                                        " event %s with empty peer info",
+                                glusterd_friend_sm_event_name_get(event_type));
+
+                                GF_FREE (event);
+                                continue;
                         }
 
                         state = glusterd_friend_state_table[peerinfo->state.state];
diff --git a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
index fe9f4e528..b8abebf06 100644
--- a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
+++ b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
@@ -188,7 +188,9 @@ glusterd3_1_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
         ret = glusterd_friend_find (rsp.uuid, rsp.hostname, &peerinfo);
 
         if (ret) {
-                GF_ASSERT (0);
+                gf_log ("", GF_LOG_ERROR, "received friend add response from"
+                        " unknown peer uuid: %s", str);
+                goto out;
         }
 
         if (op_ret)
@@ -220,6 +222,7 @@ glusterd3_1_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
         if (ret)
                 goto out;
 
+out:
         ctx = ((call_frame_t *)myframe)->local;
         ((call_frame_t *)myframe)->local = NULL;
 
@@ -234,7 +237,6 @@ glusterd3_1_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
         }
         if (ctx)
                 glusterd_destroy_probe_ctx (ctx);
-out:
         if (rsp.hostname)
                 free (rsp.hostname);//malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
@@ -285,9 +287,6 @@ glusterd3_1_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
                 "Received %s from uuid: %s, host: %s, port: %d",
                 (op_ret)?"RJT":"ACC", str, rsp.hostname, rsp.port);
 
-        if (op_ret)
-                goto respond;
-
 inject:
         ret = glusterd_friend_find (rsp.uuid, ctx->hostname, &peerinfo);
 
@@ -644,6 +643,14 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
                 free (rsp.op_errstr); //malloced by xdr
+        if (dict) {
+                if (!dict->extra_stdfree && rsp.dict.dict_val)
+                        free (rsp.dict.dict_val); //malloced by xdr
+                dict_unref (dict);
+        } else {
+                if (rsp.dict.dict_val)
+                        free (rsp.dict.dict_val); //malloced by xdr
+        }
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }