libgfapi: Wait for GF_EVENT_CHILD_DOWN in glfs_fini()

Whenever glfs_fini() is being called, currently no
check is made inside the function to determine whether
the child is already down or not. This patch will wait
for GF_EVENT_CHILD_DOWN for the active subvol and
then exits.

TBD:
Apart from the active subvol, wait for other CHILD_DOWN
events generated through operations like volume set in
future.

Change-Id: I81c64ac07b463bfed48bf306f9e8f46ba0f0a76f
BUG: 1153610
Signed-off-by: Anoop C S <achiraya@redhat.com>
Reviewed-on: http://review.gluster.org/9060
Reviewed-by: Shyamsundar Ranganathan <srangana@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-by: Niels de Vos <ndevos@redhat.com>

3 files changed, 61 insertions, 15 deletions

diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index afcbb4553ba..cb17669fd82 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -89,6 +89,7 @@ struct glfs {
 	glfs_init_cbk       init_cbk;
 	pthread_mutex_t     mutex;
 	pthread_cond_t      cond;
+        pthread_cond_t      child_down_cond; /* for broadcasting CHILD_DOWN */
 	int                 init;
 	int                 ret;
 	int                 err;
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index 89017bab84d..edf9aae37e9 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -99,10 +99,21 @@ notify (xlator_t *this, int event, void *data, ...)
 			graph->id);
 		break;
 	case GF_EVENT_CHILD_UP:
+                pthread_mutex_lock (&fs->mutex);
+                {
+                        graph->used = 1;
+                }
+                pthread_mutex_unlock (&fs->mutex);
 		graph_setup (fs, graph);
 		glfs_init_done (fs, 0);
 		break;
 	case GF_EVENT_CHILD_DOWN:
+                pthread_mutex_lock (&fs->mutex);
+                {
+                        graph->used = 0;
+                        pthread_cond_broadcast (&fs->child_down_cond);
+                }
+                pthread_mutex_unlock (&fs->mutex);
 		graph_setup (fs, graph);
 		glfs_init_done (fs, 1);
 		break;
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 12c4ea4aebb..0dc535eb9e9 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -574,6 +574,7 @@ pub_glfs_new (const char *volname)
 
 	pthread_mutex_init (&fs->mutex, NULL);
 	pthread_cond_init (&fs->cond, NULL);
+        pthread_cond_init (&fs->child_down_cond, NULL);
 
 	INIT_LIST_HEAD (&fs->openfds);
 
@@ -615,6 +616,7 @@ priv_glfs_free_from_ctx (struct glfs *fs)
                 return;
 
         (void) pthread_cond_destroy (&fs->cond);
+        (void) pthread_cond_destroy (&fs->child_down_cond);
 
         (void) pthread_mutex_destroy (&fs->mutex);
 
@@ -802,15 +804,16 @@ priv_glfs_active_subvol (struct glfs *);
 int
 pub_glfs_fini (struct glfs *fs)
 {
-        int             ret = -1;
-        int             countdown = 100;
-        xlator_t        *subvol = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-        call_pool_t     *call_pool = NULL;
-        int             fs_init = 0;
+        int                ret = -1;
+        int                countdown = 100;
+        xlator_t           *subvol = NULL;
+        glusterfs_ctx_t    *ctx = NULL;
+        glusterfs_graph_t  *graph = NULL;
+        call_pool_t        *call_pool = NULL;
+        int                fs_init = 0;
+        int                err = -1;
 
         ctx = fs->ctx;
-
         if (ctx->mgmt) {
                 rpc_clnt_disable (ctx->mgmt);
                 ctx->mgmt = NULL;
@@ -841,24 +844,55 @@ pub_glfs_fini (struct glfs *fs)
         if (fs_init != 0) {
                 subvol = priv_glfs_active_subvol (fs);
                 if (subvol) {
-                        /* PARENT_DOWN within priv_glfs_subvol_done() is issued only
-                           on graph switch (new graph should activiate and
-                           decrement the extra @winds count taken in glfs_graph_setup()
+                        /* PARENT_DOWN within glfs_subvol_done() is issued
+                           only on graph switch (new graph should activiate
+                           and decrement the extra @winds count taken in
+                           glfs_graph_setup()
 
-                           Since we are explicitly destroying, PARENT_DOWN is necessary
+                           Since we are explicitly destroying,
+                           PARENT_DOWN is necessary
                         */
                         xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
-                        /* TBD: wait for CHILD_DOWN before exiting, in case of
-                           asynchronous cleanup like graceful socket
-                           disconnection in the future.
+                        /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
+                           in case of asynchrnous cleanup
                         */
+                        graph = subvol->graph;
+                        err = pthread_mutex_lock (&fs->mutex);
+                        if (err != 0) {
+                                gf_log ("glfs", GF_LOG_ERROR,
+                                        "pthread lock on glfs mutex, "
+                                        "returned error: (%s)", strerror (err));
+                                goto fail;
+                        }
+                        /* check and wait for CHILD_DOWN for active subvol*/
+                        {
+                                while (graph->used) {
+                                        err = pthread_cond_wait (&fs->child_down_cond,
+                                                                 &fs->mutex);
+                                        if (err != 0)
+                                                gf_log ("glfs", GF_LOG_INFO,
+                                                        "%s cond wait failed %s",
+                                                        subvol->name,
+                                                        strerror (err));
+                                }
+                        }
+
+                        err = pthread_mutex_unlock (&fs->mutex);
+                        if (err != 0) {
+                                gf_log ("glfs", GF_LOG_ERROR,
+                                        "pthread unlock on glfs mutex, "
+                                        "returned error: (%s)", strerror (err));
+                                goto fail;
+                        }
                 }
                 priv_glfs_subvol_done (fs, subvol);
         }
 
         if (gf_log_fini(ctx) != 0)
                 ret = -1;
-
+fail:
+        if (!ret)
+                ret = err;
         return ret;
 }