glusterd,glusterfsd : Perform proper cleanup of connections and socket files

Perform proper disconnections and cleanup in glusterd on stopping nfs-server and
self-heal daemon processes to prevent glusterd from opening unneded unix domain
socket connections.

glusterfsd processes will cleanup the socket files properly in
cleanup_and_exit() to prevent junk socket files.

Also, fix rpcsvc_program_unregister() to preform the unregistering properly.

Change-Id: I1c7302c1166cf43feba1c7a813c3dc10169dc53a
BUG: 810089
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.com/3168
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kp@gluster.com>

4 files changed, 130 insertions, 45 deletions

diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 814de1d9349..dac92d67fb4 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -192,37 +192,9 @@ glusterfs_terminate_response_send (rpcsvc_request_t *req, int op_ret)
 }
 
 int
-glusterfs_listener_stop (void)
-{
-        glusterfs_ctx_t  *ctx = NULL;
-        cmd_args_t       *cmd_args = NULL;
-        int              ret = 0;
-        xlator_t         *this = NULL;
-
-        ctx = glusterfs_ctx_get ();
-        GF_ASSERT (ctx);
-        cmd_args = &ctx->cmd_args;
-        if (cmd_args->sock_file) {
-                ret = unlink (cmd_args->sock_file);
-                if (ret && (ENOENT == errno)) {
-                        ret = 0;
-                }
-        }
-
-        if (ret) {
-                this = THIS;
-                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink linstener "
-                        "socket %s, error: %s", cmd_args->sock_file,
-                        strerror (errno));
-        }
-        return ret;
-}
-
-int
 glusterfs_handle_terminate (rpcsvc_request_t *req)
 {
 
-        (void) glusterfs_listener_stop ();
         glusterfs_terminate_response_send (req, 0);
         cleanup_and_exit (SIGTERM);
         return 0;
@@ -1792,6 +1764,48 @@ out:
 }
 
 int
+glusterfs_listener_stop (glusterfs_ctx_t *ctx)
+{
+        cmd_args_t              *cmd_args = NULL;
+        rpcsvc_t                *rpc = NULL;
+        rpcsvc_listener_t       *listener = NULL;
+        rpcsvc_listener_t       *next = NULL;
+        int                     ret = 0;
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (ctx);
+
+        rpc = ctx->listener;
+        ctx->listener = NULL;
+
+        (void) rpcsvc_program_unregister(rpc, &glusterfs_mop_prog);
+
+        list_for_each_entry_safe (listener, next, &rpc->listeners, list) {
+                rpcsvc_listener_destroy (listener);
+        }
+
+        (void) rpcsvc_unregister_notify (rpc, glusterfs_rpcsvc_notify, THIS);
+
+        GF_FREE (rpc);
+
+        cmd_args = &ctx->cmd_args;
+        if (cmd_args->sock_file) {
+                ret = unlink (cmd_args->sock_file);
+                if (ret && (ENOENT == errno)) {
+                        ret = 0;
+                }
+        }
+
+        if (ret) {
+                this = THIS;
+                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink linstener "
+                        "socket %s, error: %s", cmd_args->sock_file,
+                        strerror (errno));
+        }
+        return ret;
+}
+
+int
 glusterfs_mgmt_init (glusterfs_ctx_t *ctx)
 {
         cmd_args_t              *cmd_args = NULL;
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 7bc063b2fee..4ad47ec621a 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -198,6 +198,7 @@ int glusterfs_pidfile_cleanup (glusterfs_ctx_t *ctx);
 int glusterfs_volumes_init (glusterfs_ctx_t *ctx);
 int glusterfs_mgmt_init (glusterfs_ctx_t *ctx);
 int glusterfs_listener_init (glusterfs_ctx_t *ctx);
+int glusterfs_listener_stop (glusterfs_ctx_t *ctx);
 
 int
 create_fuse_mount (glusterfs_ctx_t *ctx)
@@ -798,7 +799,7 @@ cleanup_and_exit (int signum)
         ctx->cleanup_started = 1;
         glusterfs_mgmt_pmap_signout (ctx);
         if (ctx->listener) {
-                ctx->listener = NULL;
+                (void) glusterfs_listener_stop (ctx);
         }
 
         /* Call fini() of FUSE xlator first:
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index b64c43f0e75..f87338315af 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1273,28 +1273,44 @@ rpcsvc_submit_message (rpcsvc_request_t *req, struct iovec *proghdr,
 
 
 int
-rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *prog)
+rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *program)
 {
         int                     ret = -1;
-
-        if (!svc || !prog) {
+        rpcsvc_program_t        *prog = NULL;
+        if (!svc || !program) {
                 goto out;
         }
 
-        ret = rpcsvc_program_unregister_portmap (prog);
+        ret = rpcsvc_program_unregister_portmap (program);
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "portmap unregistration of"
                         " program failed");
                 goto out;
         }
 
+        pthread_mutex_lock (&svc->rpclock);
+        {
+                list_for_each_entry (prog, &svc->programs, program) {
+                        if ((prog->prognum == program->prognum)
+                            && (prog->progver == program->progver)) {
+                                break;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&svc->rpclock);
+
+        if (prog == NULL) {
+                ret = -1;
+                goto out;
+        }
+
         gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Program unregistered: %s, Num: %d,"
                 " Ver: %d, Port: %d", prog->progname, prog->prognum,
                 prog->progver, prog->progport);
 
         pthread_mutex_lock (&svc->rpclock);
         {
-                list_del (&prog->program);
+                list_del_init (&prog->program);
         }
         pthread_mutex_unlock (&svc->rpclock);
 
@@ -1302,8 +1318,8 @@ rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *prog)
 out:
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "Program unregistration failed"
-                        ": %s, Num: %d, Ver: %d, Port: %d", prog->progname,
-                        prog->prognum, prog->progver, prog->progport);
+                        ": %s, Num: %d, Ver: %d, Port: %d", program->progname,
+                        program->prognum, program->progver, program->progport);
         }
 
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index ed7ba1d51e6..73770bb76a9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -2783,20 +2783,41 @@ glusterd_nodesvc_connect (char *server, char *socketpath) {
         dict_t                  *options = NULL;
         struct rpc_clnt         *rpc = NULL;
 
-        ret = rpc_clnt_transport_unix_options_build (&options, socketpath);
-        if (ret)
-                goto out;
-        ret = glusterd_rpc_create (&rpc, options,
-                                   glusterd_nodesvc_rpc_notify,
-                                   server);
-        if (ret)
-                goto out;
-        (void) glusterd_nodesvc_set_rpc (server, rpc);
+        rpc = glusterd_nodesvc_get_rpc (server);
+
+        if (rpc == NULL) {
+                ret = rpc_clnt_transport_unix_options_build (&options,
+                                                             socketpath);
+                if (ret)
+                        goto out;
+                ret = glusterd_rpc_create (&rpc, options,
+                                           glusterd_nodesvc_rpc_notify,
+                                           server);
+                if (ret)
+                        goto out;
+                (void) glusterd_nodesvc_set_rpc (server, rpc);
+        }
 out:
         return ret;
 }
 
 int32_t
+glusterd_nodesvc_disconnect (char *server)
+{
+        struct rpc_clnt         *rpc = NULL;
+
+        rpc = glusterd_nodesvc_get_rpc (server);
+
+        if (rpc) {
+                rpc_clnt_connection_cleanup (&rpc->conn);
+                rpc_clnt_unref (rpc);
+                (void)glusterd_nodesvc_set_rpc (server, NULL);
+        }
+
+        return 0;
+}
+
+int32_t
 glusterd_nodesvc_start (char *server)
 {
         int32_t                 ret                        = -1;
@@ -2910,6 +2931,31 @@ glusterd_is_nodesvc_running (char *server)
 }
 
 int32_t
+glusterd_nodesvc_unlink_socket_file (char *server)
+{
+        int             ret = 0;
+        char            sockfpath[PATH_MAX] = {0,};
+        char            rundir[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = THIS->private;
+
+        glusterd_get_nodesvc_rundir (server, priv->workdir,
+                                     rundir, sizeof (rundir));
+
+        glusterd_nodesvc_set_socket_filepath (rundir, priv->uuid,
+                                              sockfpath, sizeof (sockfpath));
+
+        ret = unlink (sockfpath);
+        if (ret && (ENOENT == errno)) {
+                ret = 0;
+        } else {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to remove %s"
+                        " error: %s", sockfpath, strerror (errno));
+        }
+
+        return ret;
+}
+
+int32_t
 glusterd_nodesvc_stop (char *server, int sig)
 {
         char                    pidfile[PATH_MAX] = {0,};
@@ -2918,9 +2964,17 @@ glusterd_nodesvc_stop (char *server, int sig)
 
         if (!glusterd_is_nodesvc_running (server))
                 goto out;
+
+        (void)glusterd_nodesvc_disconnect (server);
+
         glusterd_get_nodesvc_pidfile (server, priv->workdir,
                                             pidfile, sizeof (pidfile));
         ret = glusterd_service_stop (server, pidfile, sig, _gf_true);
+
+        if (ret == 0) {
+                glusterd_nodesvc_set_running (server, _gf_false);
+                (void)glusterd_nodesvc_unlink_socket_file (server);
+        }
 out:
         return ret;
 }