gluster: Sometimes Brick process is crashed at the time of stopping brick

Problem: Sometimes brick process is getting crashed at the time
         of stop brick while brick mux is enabled.

Solution: Brick process was getting crashed because of rpc connection
          was not cleaning properly while brick mux is enabled.In this patch
          after sending GF_EVENT_CLEANUP notification to xlator(server)
          waits for all rpc client connection destroy for specific xlator.Once rpc
          connections are destroyed in server_rpc_notify for all associated client
          for that brick then call xlator_mem_cleanup for for brick xlator as well as
          all child xlators.To avoid races at the time of cleanup introduce
          two new flags at each xlator cleanup_starting, call_cleanup.

BUG: 1544090
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Note: Run all test-cases in separate build (https://review.gluster.org/#/c/19700/)
      with same patch after enable brick mux forcefully, all test cases are
      passed.

Change-Id: Ic4ab9c128df282d146cf1135640281fcb31997bf
updates: bz#1544090

1 files changed, 46 insertions, 15 deletions

diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c
index 34b8061425c..412a47b9383 100644
--- a/libglusterfs/src/statedump.c
+++ b/libglusterfs/src/statedump.c
@@ -501,21 +501,14 @@ gf_proc_dump_dict_info (glusterfs_ctx_t *ctx)
                             (total_pairs / total_dicts));
 }
 
-void
-gf_proc_dump_xlator_info (xlator_t *top)
+static void
+gf_proc_dump_per_xlator_info (xlator_t *top)
 {
-        xlator_t        *trav = NULL;
-        glusterfs_ctx_t *ctx = NULL;
+        xlator_t        *trav = top;
+        glusterfs_ctx_t *ctx = top->ctx;
         char             itable_key[1024] = {0,};
 
-        if (!top)
-                return;
-
-        ctx = top->ctx;
-
-        trav = top;
-        while (trav) {
-
+        while (trav && !trav->cleanup_starting) {
                 if (ctx->measure_latency)
                         gf_proc_dump_latency_info (trav);
 
@@ -539,7 +532,6 @@ gf_proc_dump_xlator_info (xlator_t *top)
                 if (GF_PROC_DUMP_IS_XL_OPTION_ENABLED (inode) &&
                     (trav->dumpops->inode))
                         trav->dumpops->inode (trav);
-
                 if (trav->dumpops->fd &&
                     GF_PROC_DUMP_IS_XL_OPTION_ENABLED (fd))
                         trav->dumpops->fd (trav);
@@ -550,6 +542,30 @@ gf_proc_dump_xlator_info (xlator_t *top)
 
                 trav = trav->next;
         }
+}
+
+
+
+void
+gf_proc_dump_xlator_info (xlator_t *top, gf_boolean_t brick_mux)
+{
+        xlator_t        *trav = NULL;
+        xlator_list_t   **trav_p        = NULL;
+
+        if (!top)
+                return;
+
+        trav = top;
+        gf_proc_dump_per_xlator_info (trav);
+
+        if (brick_mux) {
+                trav_p = &top->children;
+                while (*trav_p) {
+                        trav = (*trav_p)->xlator;
+                        gf_proc_dump_per_xlator_info (trav);
+                        trav_p = &(*trav_p)->next;
+                }
+        }
 
         return;
 }
@@ -803,12 +819,27 @@ gf_proc_dump_info (int signum, glusterfs_ctx_t *ctx)
         char               tmp_dump_name[PATH_MAX] = {0,};
         char               path[PATH_MAX]          = {0,};
         struct timeval     tv                      = {0,};
+        gf_boolean_t       is_brick_mux            = _gf_false;
+        xlator_t          *top                     = NULL;
+        xlator_list_t    **trav_p                 = NULL;
+        int                brick_count            = 0;
 
         gf_proc_dump_lock ();
 
         if (!ctx)
                 goto out;
 
+        if (ctx) {
+                top = ctx->active->first;
+                for (trav_p = &top->children; *trav_p;
+                                     trav_p = &(*trav_p)->next) {
+                        brick_count++;
+                }
+
+                if (brick_count > 1)
+                        is_brick_mux = _gf_true;
+        }
+
         if (ctx->cmd_args.brick_name) {
                 GF_REMOVE_SLASH_FROM_PATH (ctx->cmd_args.brick_name, brick_name);
         } else
@@ -868,12 +899,12 @@ gf_proc_dump_info (int signum, glusterfs_ctx_t *ctx)
 
         if (ctx->master) {
                 gf_proc_dump_add_section ("fuse");
-                gf_proc_dump_xlator_info (ctx->master);
+                gf_proc_dump_xlator_info (ctx->master, _gf_false);
         }
 
         if (ctx->active) {
                 gf_proc_dump_add_section ("active graph - %d", ctx->graph_id);
-                gf_proc_dump_xlator_info (ctx->active->top);
+                gf_proc_dump_xlator_info (ctx->active->top, is_brick_mux);
         }
 
         i = 0;