mgmt/glusterd: Consider nodesvc to be running after online

Definition of online in the message below is that the
RPC_CLNT_CONNECT event arrives for the nfs/self-heal-daemon process.

For automated tests, sometimes the script needs to wait until
self-heal-daemon comes online, so that the relevant
commands can be executed. Gluster volume status before this change
printed whether the self-heal-daemon is running or not based on the
lock availability on the pidfile. But there is a small window where
the lock on pid file is present but the process is still not
online. So the commands that were depending on this kept failing in
the test script.

Change-Id: I0e44e18b08d7b653d34fa170c1f187d91c888cd9
BUG: 858212
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/4236
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

7 files changed, 23 insertions, 21 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 1b1d113b578..673bfaeb70f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -2997,14 +2997,14 @@ glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
         switch (event) {
         case RPC_CLNT_CONNECT:
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
-                (void) glusterd_nodesvc_set_running (server, _gf_true);
+                (void) glusterd_nodesvc_set_online_status (server, _gf_true);
                 ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
                 break;
 
         case RPC_CLNT_DISCONNECT:
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT");
-                (void) glusterd_nodesvc_set_running (server, _gf_false);
+                (void) glusterd_nodesvc_set_online_status (server, _gf_false);
                 break;
 
         default:
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index ac53f2e463b..48ae5b660db 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -4134,7 +4134,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
         case GF_CLI_STATS_INFO:
                 ret = dict_get_str_boolean (dict, "nfs", _gf_false);
                 if (ret) {
-                        if (!glusterd_nodesvc_is_running ("nfs")) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
                                 ret = -1;
                                 gf_log (this->name, GF_LOG_ERROR, "NFS server"
                                         " is not running");
@@ -4177,7 +4177,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
         case GF_CLI_STATS_TOP:
                 ret = dict_get_str_boolean (dict, "nfs", _gf_false);
                 if (ret) {
-                        if (!glusterd_nodesvc_is_running ("nfs")) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
                                 ret = -1;
                                 gf_log (this->name, GF_LOG_ERROR, "NFS server"
                                         " is not running");
@@ -4614,7 +4614,7 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
 
                 ret = 0;
         } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
-                if (!glusterd_nodesvc_is_running ("nfs")) {
+                if (!glusterd_is_nodesvc_online ("nfs")) {
                         ret = -1;
                         gf_log (this->name, GF_LOG_ERROR,
                                 "NFS server is not running");
@@ -4633,7 +4633,7 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
 
                 ret = 0;
         } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
-                if (!glusterd_nodesvc_is_running ("glustershd")) {
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
                         ret = -1;
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Self-heal daemon is not running");
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 5395e0a9cc7..879bb126a30 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -3090,7 +3090,7 @@ glusterd_get_nodesvc_volfile (char *server, char *workdir,
 }
 
 void
-glusterd_nodesvc_set_running (char *server, gf_boolean_t status)
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status)
 {
         glusterd_conf_t *priv = NULL;
 
@@ -3101,16 +3101,16 @@ glusterd_nodesvc_set_running (char *server, gf_boolean_t status)
         GF_ASSERT (priv->nfs);
 
         if (!strcmp("glustershd", server))
-                priv->shd->running = status;
+                priv->shd->online = status;
         else if (!strcmp ("nfs", server))
-                priv->nfs->running = status;
+                priv->nfs->online = status;
 }
 
 gf_boolean_t
-glusterd_nodesvc_is_running (char *server)
+glusterd_is_nodesvc_online (char *server)
 {
         glusterd_conf_t *conf = NULL;
-        gf_boolean_t    running = _gf_false;
+        gf_boolean_t    online = _gf_false;
 
         GF_ASSERT (server);
         conf = THIS->private;
@@ -3119,11 +3119,11 @@ glusterd_nodesvc_is_running (char *server)
         GF_ASSERT (conf->nfs);
 
         if (!strcmp (server, "glustershd"))
-                running = conf->shd->running;
+                online = conf->shd->online;
         else if (!strcmp (server, "nfs"))
-                running = conf->nfs->running;
+                online = conf->nfs->online;
 
-        return running;
+        return online;
 }
 
 int32_t
@@ -3418,7 +3418,7 @@ glusterd_nodesvc_stop (char *server, int sig)
         ret = glusterd_service_stop (server, pidfile, sig, _gf_true);
 
         if (ret == 0) {
-                glusterd_nodesvc_set_running (server, _gf_false);
+                glusterd_nodesvc_set_online_status (server, _gf_false);
                 (void)glusterd_nodesvc_unlink_socket_file (server);
         }
 out:
@@ -3492,7 +3492,9 @@ glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
 
         glusterd_get_nodesvc_pidfile (server, priv->workdir, pidfile,
                                       sizeof (pidfile));
-        running = glusterd_is_service_running (pidfile, &pid);
+        //Consider service to be running only when glusterd sees it Online
+        if (glusterd_is_nodesvc_online (server))
+                running = glusterd_is_service_running (pidfile, &pid);
 
         /* For nfs-servers/self-heal-daemon setting
          * brick<n>.hostname = "NFS Server" / "Self-heal Daemon"
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 00353b03e03..c90c9d9182e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -208,10 +208,10 @@ int32_t
 glusterd_nodesvc_connect (char *server, char *socketpath);
 
 void
-glusterd_nodesvc_set_running (char *server, gf_boolean_t status);
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status);
 
 gf_boolean_t
-glusterd_nodesvc_is_running (char *server);
+glusterd_is_nodesvc_online (char *server);
 
 int
 glusterd_remote_hostname_get (rpcsvc_request_t *req,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index db143d5f07a..263c8bf013b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1102,7 +1102,7 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        if (!glusterd_nodesvc_is_running ("glustershd")) {
+        if (!glusterd_is_nodesvc_online ("glustershd")) {
                 ret = -1;
                 snprintf (msg, sizeof (msg), "Self-heal daemon is not "
                           "running. Check self-heal daemon log file.");
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index f9cdb65ae55..204a6adfce5 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1057,7 +1057,7 @@ init (xlator_t *this)
 #endif
 
         this->private = conf;
-        (void) glusterd_nodesvc_set_running ("glustershd", _gf_false);
+        (void) glusterd_nodesvc_set_online_status ("glustershd", _gf_false);
 
         GLUSTERD_GET_HOOKS_DIR (hooks_dir, GLUSTERD_HOOK_VER, conf);
         if (stat (hooks_dir, &buf)) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 36cd8f8fc56..66adff741bf 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -96,7 +96,7 @@ struct glusterd_volgen {
 
 typedef struct {
         struct rpc_clnt         *rpc;
-        gf_boolean_t            running;
+        gf_boolean_t            online;
 } nodesrv_t;
 
 #define GD_OP_VERSION_KEY     "operating-version"