protocol client/server: Fix client-server handshake

Problem:
Currently on a successful connection between protocol
server and client, the protocol client initiates a
CHILD_UP event in the client stack. At this point in
time, only the connection between server and client is
established, and there is no guarantee that the server
side stack is ready to serve requests.

It works fine now, as most server side translators are
not dependent on any other factors, before being able
to serve requests today and hence they are up by the time
the client stack translators receive the CHILD_UP (initiated
by client handshake).

The gap here is exposed when certain server side translators
like NSR-Server for example, have a couple of protocol clients
as their child(connecting them to other bricks), and they
can't really serve requests till a quorum of their children are
up. Hence these translators should defer sending CHILD_UP
till they have enough children up, and the same needs to be
propagated to the client stack translators.

Fix:
Maintain a child_up variable in both the protocol client
and protocol server translators. The protocol server should
update this value based on the CHILD_UP and CHILD_DOWN
events it receives from the translators below it. On receiving
such an event it should forward that event to the client.
The protocol client on receiving such an event should forward
it up the client stack, thereby letting the client translators
correctly know that the server is up and ready to serve.

The clients connecting later(long after a server has initialized
and processed it's CHILD_UP events), will receive a child_up status
as part of the handshake, and based on the status of the server's
child_up, can either propagate a CHILD_UP event or defer it.

Change-Id: I0807141e62118d8de9d9cde57a53a607be44a0e0
BUG: 1312845
Signed-off-by: Avra Sengupta <asengupt@redhat.com>
Reviewed-on: http://review.gluster.org/13549
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

3 files changed, 78 insertions, 1 deletions

diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index 293509c5d3f..fe5dfbab516 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -385,6 +385,12 @@ server_setvolume (rpcsvc_request_t *req)
                 goto fail;
         }
 
+        ret = dict_set_int32 (reply, "child_up", conf->child_up);
+        if (ret < 0)
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        PS_MSG_DICT_GET_FAILED, "Failed to set 'child_up' "
+                        "in the reply dict");
+
         buf = memdup (args.dict.dict_val, args.dict.dict_len);
         if (buf == NULL) {
                 op_ret = -1;
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index d07b840b3fc..24e31500453 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -958,6 +958,8 @@ init (xlator_t *this)
         if (ret)
                 conf->conf_dir = CONFDIR;
 
+        conf->child_up = _gf_false;
+
         /*ret = dict_get_str (this->options, "statedump-path", &statedump_path);
         if (!ret) {
                 gf_path_strip_trailing_slashes (statedump_path);
@@ -1237,6 +1239,35 @@ out:
 }
 
 int
+server_process_child_event (xlator_t *this, int32_t event, void *data,
+                            enum gf_cbk_procnum cbk_procnum)
+{
+        int              ret          = -1;
+        server_conf_t    *conf        = NULL;
+        rpc_transport_t  *xprt        = NULL;
+
+        GF_VALIDATE_OR_GOTO(this->name, data, out);
+
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+        pthread_mutex_lock (&conf->mutex);
+        {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        rpcsvc_callback_submit (conf->rpc, xprt,
+                                                &server_cbk_prog,
+                                                cbk_procnum,
+                                                NULL, 0);
+                }
+        }
+        pthread_mutex_unlock (&conf->mutex);
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
         int              ret          = -1;
@@ -1246,6 +1277,10 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         server_conf_t    *conf        = NULL;
         va_list          ap;
 
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
         dict = data;
         va_start (ap, data);
         output = va_arg (ap, dict_t*);
@@ -1272,7 +1307,41 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 
                 conf->parent_up = _gf_true;
 
-                /* fall through and notify the event to children */
+                default_notify (this, event, data);
+                break;
+        }
+
+        case GF_EVENT_CHILD_UP:
+        {
+                conf->child_up = _gf_true;
+                ret = server_process_child_event (this, event, data,
+                                                  GF_CBK_CHILD_UP);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                PS_MSG_SERVER_EVENT_UPCALL_FAILED,
+                                "server_process_child_event failed");
+                        goto out;
+                }
+
+                default_notify (this, event, data);
+                break;
+        }
+
+        case GF_EVENT_CHILD_DOWN:
+        {
+                conf->child_up = _gf_false;
+                ret = server_process_child_event (this, event, data,
+                                                  GF_CBK_CHILD_DOWN);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                PS_MSG_SERVER_EVENT_UPCALL_FAILED,
+                                "server_process_child_event failed");
+                        goto out;
+                }
+
+                default_notify (this, event, data);
+                break;
+
         }
 
         default:
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index c0e2752cad7..7980d300be7 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -72,6 +72,8 @@ struct server_conf {
                                             * in case if volume set options
                                             * (say *.allow | *.reject) are
                                             * tweeked */
+        gf_boolean_t            child_up; /* Set to true, when child is up, and
+                                           * false, when child is down */
 };
 typedef struct server_conf server_conf_t;