glusterd, cli: Enable errstr for peer probe

Presently glusterd only returns an errno to cli for peer probe command. This
patch allows glusterd to return an errstr as well to cli. An op_errstr member
has been added to gf1_cli_probe_rsp and gd1_mgmt_probe_rsp structs to allow
this.
In case of an error, cli will display the errstr if it was set. If errstr is not
set cli will display the error message based on errno.
Also, to allow for return of errstr in cases such as handshake failure, an
errstr member has been added to the glusterd_peerctx_t struct.

Change-Id: Iece2b44a7181555e960d9fe4517ec6cda4cdb385
BUG: 816840
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.com/3262
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

13 files changed, 148 insertions, 53 deletions

diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 434e9a6b13c..48c1b971820 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -136,41 +136,48 @@ gf_cli3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
          }
 
         if (rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_PROBE_ANOTHER_CLUSTER:
-                                snprintf (msg, sizeof (msg),
-                                          "%s is already part of another"
-                                          " cluster", rsp.hostname);
-                                break;
-                        case GF_PROBE_VOLUME_CONFLICT:
-                                snprintf (msg, sizeof (msg),
-                                          "Atleast one volume on %s conflicts "
-                                          "with existing volumes in the "
-                                          "cluster", rsp.hostname);
-                                break;
-                        case GF_PROBE_UNKNOWN_PEER:
-                                snprintf (msg, sizeof (msg),
-                                          "%s responded with 'unknown peer'"
-                                          " error, this could happen if %s "
-                                          "doesn't have localhost in its peer"
-                                          " database", rsp.hostname,
-                                          rsp.hostname);
-                                break;
-                        case GF_PROBE_ADD_FAILED:
-                                snprintf (msg, sizeof (msg),
-                                          "Failed to add peer information "
-                                          "on %s" , rsp.hostname);
-                                break;
-
-                        default:
-                                snprintf (msg, sizeof (msg),
-                                          "Probe unsuccessful\nProbe returned "
-                                          "with unknown errno %d",
-                                          rsp.op_errno);
-                                break;
+                if (rsp.op_errstr && (strlen (rsp.op_errstr) > 0)) {
+                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                } else {
+                        switch (rsp.op_errno) {
+                                case GF_PROBE_ANOTHER_CLUSTER:
+                                        snprintf (msg, sizeof (msg),
+                                                  "%s is already part of "
+                                                  "another cluster",
+                                                  rsp.hostname);
+                                        break;
+                                case GF_PROBE_VOLUME_CONFLICT:
+                                        snprintf (msg, sizeof (msg),
+                                                  "Atleast one volume on %s "
+                                                  "conflicts with existing "
+                                                  "volumes in the cluster",
+                                                  rsp.hostname);
+                                        break;
+                                case GF_PROBE_UNKNOWN_PEER:
+                                        snprintf (msg, sizeof (msg),
+                                                  "%s responded with 'unknown "
+                                                  "peer' error, this could "
+                                                  "happen if %s doesn't have "
+                                                  "localhost in its peer "
+                                                  "database", rsp.hostname,
+                                                  rsp.hostname);
+                                        break;
+                                case GF_PROBE_ADD_FAILED:
+                                        snprintf (msg, sizeof (msg),
+                                                  "Failed to add peer "
+                                                  "information on %s" ,
+                                                  rsp.hostname);
+                                        break;
+
+                                default:
+                                        snprintf (msg, sizeof (msg),
+                                                  "Probe unsuccessful\nProbe "
+                                                  "returned with unknown errno "
+                                                  "%d", rsp.op_errno);
+                                        break;
+                        }
                 }
-                gf_log ("glusterd",GF_LOG_ERROR,"Probe failed with op_ret %d"
-                        " and op_errno %d", rsp.op_ret, rsp.op_errno);
+                gf_log ("cli", GF_LOG_ERROR, "%s", msg);
         }
 
 #if (HAVE_LIB_XML)
diff --git a/rpc/xdr/src/cli1-xdr.c b/rpc/xdr/src/cli1-xdr.c
index 56562ee3d00..607c36b3ec2 100644
--- a/rpc/xdr/src/cli1-xdr.c
+++ b/rpc/xdr/src/cli1-xdr.c
@@ -252,6 +252,8 @@ xdr_gf1_cli_probe_rsp (XDR *xdrs, gf1_cli_probe_rsp *objp)
 		}
 		 if (!xdr_string (xdrs, &objp->hostname, ~0))
 			 return FALSE;
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
 		return TRUE;
 	} else if (xdrs->x_op == XDR_DECODE) {
 		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
@@ -270,6 +272,8 @@ xdr_gf1_cli_probe_rsp (XDR *xdrs, gf1_cli_probe_rsp *objp)
 		}
 		 if (!xdr_string (xdrs, &objp->hostname, ~0))
 			 return FALSE;
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
 	 return TRUE;
 	}
 
@@ -281,6 +285,8 @@ xdr_gf1_cli_probe_rsp (XDR *xdrs, gf1_cli_probe_rsp *objp)
 		 return FALSE;
 	 if (!xdr_string (xdrs, &objp->hostname, ~0))
 		 return FALSE;
+	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+		 return FALSE;
 	return TRUE;
 }
 
diff --git a/rpc/xdr/src/cli1-xdr.h b/rpc/xdr/src/cli1-xdr.h
index 234f0f7f797..48aa863cbc8 100644
--- a/rpc/xdr/src/cli1-xdr.h
+++ b/rpc/xdr/src/cli1-xdr.h
@@ -200,6 +200,7 @@ struct gf1_cli_probe_rsp {
 	int op_errno;
 	int port;
 	char *hostname;
+	char *op_errstr;
 };
 typedef struct gf1_cli_probe_rsp gf1_cli_probe_rsp;
 
diff --git a/rpc/xdr/src/cli1-xdr.x b/rpc/xdr/src/cli1-xdr.x
index 61af3a14cbf..750b491e582 100644
--- a/rpc/xdr/src/cli1-xdr.x
+++ b/rpc/xdr/src/cli1-xdr.x
@@ -135,6 +135,7 @@ enum gf_cli_status_type {
         int     op_errno;
 	int	port;
         string  hostname<>;
+        string  op_errstr<>;
 }  ;
 
  struct gf1_cli_deprobe_req {
diff --git a/rpc/xdr/src/glusterd1-xdr.c b/rpc/xdr/src/glusterd1-xdr.c
index a5438d23e3f..28ab49b6f73 100644
--- a/rpc/xdr/src/glusterd1-xdr.c
+++ b/rpc/xdr/src/glusterd1-xdr.c
@@ -66,6 +66,55 @@ xdr_gd1_mgmt_probe_rsp (XDR *xdrs, gd1_mgmt_probe_rsp *objp)
 	register int32_t *buf;
         buf = NULL;
 
+
+	if (xdrs->x_op == XDR_ENCODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		 if (!xdr_string (xdrs, &objp->hostname, ~0))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->port))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		IXDR_PUT_LONG(buf, objp->port);
+		IXDR_PUT_LONG(buf, objp->op_ret);
+		IXDR_PUT_LONG(buf, objp->op_errno);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		return TRUE;
+	} else if (xdrs->x_op == XDR_DECODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		 if (!xdr_string (xdrs, &objp->hostname, ~0))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->port))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		objp->port = IXDR_GET_LONG(buf);
+		objp->op_ret = IXDR_GET_LONG(buf);
+		objp->op_errno = IXDR_GET_LONG(buf);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+	 return TRUE;
+	}
+
 	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
 		sizeof (u_char), (xdrproc_t) xdr_u_char))
 		 return FALSE;
@@ -77,6 +126,8 @@ xdr_gd1_mgmt_probe_rsp (XDR *xdrs, gd1_mgmt_probe_rsp *objp)
 		 return FALSE;
 	 if (!xdr_int (xdrs, &objp->op_errno))
 		 return FALSE;
+	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+		 return FALSE;
 	return TRUE;
 }
 
diff --git a/rpc/xdr/src/glusterd1-xdr.h b/rpc/xdr/src/glusterd1-xdr.h
index 14c41baffba..89de04ebef2 100644
--- a/rpc/xdr/src/glusterd1-xdr.h
+++ b/rpc/xdr/src/glusterd1-xdr.h
@@ -62,6 +62,7 @@ struct gd1_mgmt_probe_rsp {
 	int port;
 	int op_ret;
 	int op_errno;
+	char *op_errstr;
 };
 typedef struct gd1_mgmt_probe_rsp gd1_mgmt_probe_rsp;
 
diff --git a/rpc/xdr/src/glusterd1-xdr.x b/rpc/xdr/src/glusterd1-xdr.x
index c30c71e021a..fc1bb58b4a8 100644
--- a/rpc/xdr/src/glusterd1-xdr.x
+++ b/rpc/xdr/src/glusterd1-xdr.x
@@ -16,6 +16,7 @@
         int     port;
         int     op_ret;
         int     op_errno;
+        string op_errstr<>;
 }  ;
 
 struct gd1_mgmt_friend_req {
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index d7ac4fd4f2c..0950161532a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -679,7 +679,7 @@ glusterd_handle_cli_probe (rpcsvc_request_t *req)
                 cli_req.hostname, cli_req.port);
 
         if (!(ret = glusterd_is_local_addr(cli_req.hostname))) {
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST,
+                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST, NULL,
                                               cli_req.hostname, cli_req.port);
                 goto out;
         }
@@ -691,7 +691,8 @@ glusterd_handle_cli_probe (rpcsvc_request_t *req)
                         gf_log ("glusterd", GF_LOG_DEBUG, "Probe host %s port %d"
                                " already a peer", cli_req.hostname, cli_req.port);
                         glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND,
-                                                      cli_req.hostname, cli_req.port);
+                                                      NULL, cli_req.hostname,
+                                                      cli_req.port);
                         goto out;
                 }
         }
@@ -2237,10 +2238,11 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
                         event->peerinfo = peerinfo;
                         ret = glusterd_friend_sm_inject_event (event);
                         glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_SUCCESS,
-                                                      (char*)hoststr, port);
+                                                      NULL, (char*)hoststr,
+                                                      port);
                 }
         } else {
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND,
+                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND, NULL,
                                               (char*)hoststr, port);
         }
 
@@ -2372,7 +2374,8 @@ glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *hostname, int port,
 
 int
 glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *hostname, int port)
+                              int32_t op_errno, char *op_errstr, char *hostname,
+                              int port)
 {
         gf1_cli_probe_rsp    rsp = {0, };
         int32_t              ret = -1;
@@ -2381,6 +2384,7 @@ glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
 
         rsp.op_ret = op_ret;
         rsp.op_errno = op_errno;
+        rsp.op_errstr = op_errstr ? op_errstr : "";
         rsp.hostname = hostname;
         rsp.port = port;
 
@@ -2800,10 +2804,19 @@ glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 }
 
 int
-glusterd_friend_remove_notify (glusterd_peerinfo_t *peerinfo, rpcsvc_request_t *req)
+glusterd_friend_remove_notify (glusterd_peerctx_t *peerctx)
 {
-        int ret = -1;
-        glusterd_friend_sm_event_t *new_event = NULL;
+        int                             ret = -1;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+        glusterd_peerinfo_t             *peerinfo = peerctx->peerinfo;
+        rpcsvc_request_t                *req = peerctx->args.req;
+        char                            *errstr = peerctx->errstr;
+
+        GF_ASSERT (peerctx);
+
+        peerinfo = peerctx->peerinfo;
+        req = peerctx->args.req;
+        errstr = peerctx->errstr;
 
         ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
                                             &new_event);
@@ -2815,7 +2828,7 @@ glusterd_friend_remove_notify (glusterd_peerinfo_t *peerinfo, rpcsvc_request_t *
                         goto out;
                 }
 
-                glusterd_xfer_cli_probe_resp (req, -1, ENOTCONN,
+                glusterd_xfer_cli_probe_resp (req, -1, ENOTCONN, errstr,
                                               peerinfo->hostname, peerinfo->port);
 
                 new_event->peerinfo = peerinfo;
@@ -2902,8 +2915,7 @@ glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 
                 //Inject friend disconnected here
                 if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT)  {
-                        glusterd_friend_remove_notify (peerinfo,
-                                                       peerctx->args.req);
+                        glusterd_friend_remove_notify (peerctx);
                 }
 
                 //default_notify (this, GF_EVENT_CHILD_DOWN, NULL);
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index af9c0ecd2b2..2d9dd8ada7e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -419,6 +419,7 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
         call_frame_t        *frame    = NULL;
         glusterd_peerinfo_t *peerinfo = NULL;
         glusterd_peerctx_t  *peerctx  = NULL;
+        char                msg[1024] = {0,};
 
         this = THIS;
         frame = myframe;
@@ -426,19 +427,25 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
         peerinfo = peerctx->peerinfo;
 
         if (-1 == req->rpc_status) {
-                gf_log ("", GF_LOG_ERROR,
-                        "error through RPC layer, retry again later");
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "failed to decode XDR");
+                snprintf (msg, sizeof (msg), "Failed to decode XDR");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
                 goto out;
         }
         if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get the 'versions' from remote server");
+                snprintf (msg, sizeof (msg),
+                          "Failed to get the 'versions' from remote server");
+                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
                 goto out;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index a7ccda71f72..75778929820 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -225,6 +225,7 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                 if (ctx->req) {
                         glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
                                                       rsp.op_errno,
+                                                      rsp.op_errstr,
                                                       ctx->hostname, ctx->port);
                 }
 
@@ -252,6 +253,7 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                 if (ctx->req) {
                         glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
                                                       rsp.op_errno,
+                                                      rsp.op_errstr,
                                                       ctx->hostname, ctx->port);
                 }
 
@@ -372,7 +374,8 @@ out:
 
         if (ctx->req)//reverse probe doesn't have req
                 ret = glusterd_xfer_cli_probe_resp (ctx->req, op_ret, op_errno,
-                                                    ctx->hostname, ctx->port);
+                                                    NULL, ctx->hostname,
+                                                    ctx->port);
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index 7baaf023602..e8a545c27b2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -119,6 +119,7 @@ typedef struct glusterd_peer_ctx_args_ {
 typedef struct glusterd_peer_ctx_ {
         glusterd_peerctx_args_t        args;
         glusterd_peerinfo_t            *peerinfo;
+        char                           *errstr;
 } glusterd_peerctx_t;
 
 typedef enum glusterd_friend_sm_event_type_ {
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index f04792c7b9a..ea0ee68e680 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -911,8 +911,11 @@ glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo)
                 peerinfo->rpc->mydata = NULL;
                 peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
                 peerinfo->rpc = NULL;
-                if (peerctx)
+                if (peerctx) {
+                        if (peerctx->errstr)
+                                GF_FREE (peerctx->errstr);
                         GF_FREE (peerctx);
+                }
         }
         glusterd_peer_destroy (peerinfo);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index f3e40833e39..ac9b5d03ddd 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -434,7 +434,8 @@ glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req);
 
 int
 glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *hostname, int port);
+                              int32_t op_errno, char *op_errstr, char *hostname,
+                              int port);
 
 int
 glusterd_op_commit_send_resp (rpcsvc_request_t *req,