glusterfsd: Memleak in glusterfsd process while brick mux is on

Problem: At the time of stopping the volume while brick multiplex is
         enabled memory is not cleanup from all server side xlators.

Solution: To cleanup memory for all server side xlators call fini
          in glusterfs_handle_terminate after send GF_EVENT_CLEANUP
          notification to top xlator.

BUG: 1544090
Change-Id: Ifa1525e25b697371276158705026b421b4f81140
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

16 files changed, 108 insertions, 83 deletions

diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 985c5fbc389..99e9196fd62 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -301,7 +301,6 @@ is_fop_latency_started (call_frame_t *frame)
                                                throughput, iosstat);           \
         } while (0)
 
-
 static int
 ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
 {
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.c b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
index 76a4c3f6962..830357070d6 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
@@ -229,18 +229,6 @@ notify (xlator_t *this, int event, void *data, ...)
         if (!priv)
                 return 0;
 
-        switch (event) {
-        case GF_EVENT_CLEANUP:
-                if (priv->signth) {
-                        (void) gf_thread_cleanup_xint (priv->signth);
-                        priv->signth = 0;
-                }
-                if (priv->container.thread) {
-                        (void) gf_thread_cleanup_xint (priv->container.thread);
-                        priv->container.thread = 0;
-                }
-                break;
-        }
         default_notify (this, event, data);
         return 0;
 }
@@ -263,6 +251,7 @@ fini (xlator_t *this)
                         "Could not cancel sign serializer thread");
                 goto out;
         }
+        priv->signth = 0;
 
         while (!list_empty (&priv->squeue)) {
                 sigstub = list_first_entry (&priv->squeue,
@@ -284,12 +273,19 @@ fini (xlator_t *this)
                 goto out;
         }
 
+        priv->container.thread = 0;
+
         while (!list_empty (&priv->container.bad_queue)) {
                 stub = list_first_entry (&priv->container.bad_queue, call_stub_t,
                                          list);
                 list_del_init (&stub->list);
                 call_stub_destroy (stub);
-        };
+        }
+
+        if (priv->local_pool) {
+                mem_pool_destroy (priv->local_pool);
+                priv->local_pool = NULL;
+        }
 
         pthread_mutex_destroy (&priv->container.bad_lock);
         pthread_cond_destroy (&priv->container.bad_cond);
diff --git a/xlators/features/changelog/src/changelog-rpc-common.c b/xlators/features/changelog/src/changelog-rpc-common.c
index 7bd5b688fa5..1745f218b08 100644
--- a/xlators/features/changelog/src/changelog-rpc-common.c
+++ b/xlators/features/changelog/src/changelog-rpc-common.c
@@ -275,6 +275,10 @@ changelog_rpc_server_destroy (xlator_t *this, rpcsvc_t *rpc, char *sockfile,
 
         (void) rpcsvc_unregister_notify (rpc, fn, this);
         sys_unlink (sockfile);
+        if (rpc->rxpool) {
+                mem_pool_destroy (rpc->rxpool);
+                rpc->rxpool = NULL;
+        }
 
         GF_FREE (rpc);
 }
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
index 10a149232c2..21de0fbc0c6 100644
--- a/xlators/features/changelog/src/changelog.c
+++ b/xlators/features/changelog/src/changelog.c
@@ -2111,14 +2111,6 @@ notify (xlator_t *this, int event, void *data, ...)
         if (!priv)
                 goto out;
 
-        if (event == GF_EVENT_CLEANUP) {
-               /* terminate helper threads */
-               changelog_cleanup_helper_threads (this, priv);
-
-               /* terminate RPC server/threads */
-               changelog_cleanup_rpc_threads (this, priv);
-        }
-
         if (event == GF_EVENT_TRANSLATOR_OP) {
 
                 dict = data;
@@ -2922,6 +2914,7 @@ fini (xlator_t *this)
         }
 
         this->private = NULL;
+        this->local_pool = NULL;
 
         return;
 }
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.c b/xlators/features/changetimerecorder/src/changetimerecorder.c
index a9af008f807..40bfcf4a32a 100644
--- a/xlators/features/changetimerecorder/src/changetimerecorder.c
+++ b/xlators/features/changetimerecorder/src/changetimerecorder.c
@@ -19,7 +19,6 @@
 #include "tier-ctr-interface.h"
 
 /*******************************inode forget***********************************/
-
 int
 ctr_forget (xlator_t *this, inode_t *inode)
 {
@@ -2306,15 +2305,6 @@ notify (xlator_t *this, int event, void *data, ...)
        if (!priv)
                goto out;
 
-       if (event == GF_EVENT_CLEANUP) {
-               if (fini_db (priv->_db_conn)) {
-                       gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CTR_MSG_CLOSE_DB_CONN_FAILED, "Failed closing "
-                                "db connection");
-               }
-               if (priv->_db_conn)
-                        priv->_db_conn = NULL;
-       }
        ret = default_notify (this, event, data);
 
 out:
@@ -2355,6 +2345,10 @@ fini (xlator_t *this)
                                 CTR_MSG_CLOSE_DB_CONN_FAILED, "Failed closing "
                                 "db connection");
                 }
+
+                if (priv->_db_conn)
+                        priv->_db_conn = NULL;
+
                 GF_FREE (priv->ctr_db_path);
                 if (pthread_mutex_destroy (&priv->compact_lock)) {
                         gf_msg (this->name, GF_LOG_WARNING, 0,
@@ -2364,6 +2358,7 @@ fini (xlator_t *this)
         }
         GF_FREE (priv);
         mem_pool_destroy (this->local_pool);
+        this->local_pool = NULL;
 
         return;
 }
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index ed89c02ae74..5e723e5e4c6 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -2460,6 +2460,13 @@ fini (xlator_t *this)
         priv = this->private;
         if (!priv)
                 goto out;
+
+        priv->down = _gf_true;
+        pthread_cond_broadcast (&priv->cond);
+        if (priv->thread) {
+                gf_thread_cleanup_xint (priv->thread);
+                priv->thread = 0;
+        }
         this->private = NULL;
         LOCK_DESTROY (&priv->lock);
         pthread_cond_destroy (&priv->cond);
@@ -2471,8 +2478,11 @@ fini (xlator_t *this)
         if (priv->complete_watchlist)
                 dict_unref (priv->complete_watchlist);
         GF_FREE (priv);
-        mem_pool_destroy (this->local_pool);
-        this->local_pool = NULL;
+
+        if (this->local_pool) {
+                mem_pool_destroy (this->local_pool);
+                this->local_pool = NULL;
+        }
 out:
         return;
 }
@@ -2542,13 +2552,6 @@ notify (xlator_t *this, int event, void *data, ...)
         if (!priv)
                 return 0;
 
-        switch (event) {
-        case GF_EVENT_CLEANUP:
-                priv->down = _gf_true;
-                pthread_cond_broadcast (&priv->cond);
-                break;
-        }
-
         ret = default_notify (this, event, data);
         return ret;
 }
diff --git a/xlators/features/leases/src/leases.c b/xlators/features/leases/src/leases.c
index d4fdbfb7ea1..ebee682a685 100644
--- a/xlators/features/leases/src/leases.c
+++ b/xlators/features/leases/src/leases.c
@@ -1041,14 +1041,17 @@ fini (xlator_t *this)
 
         priv->fini = _gf_true;
         pthread_cond_broadcast (&priv->cond);
-        pthread_join (priv->recall_thr, NULL);
-
-        priv->inited_recall_thr = _gf_false;
+        if (priv->recall_thr) {
+                gf_thread_cleanup_xint (priv->recall_thr);
+                priv->recall_thr = 0;
+                priv->inited_recall_thr = _gf_false;
+        }
 
         GF_FREE (priv);
-
-        glusterfs_ctx_tw_put (this->ctx);
-
+        if (this->ctx->tw) {
+                glusterfs_ctx_tw_put (this->ctx);
+                this->ctx->tw = NULL;
+        }
         return 0;
 }
 
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index cb8f8356259..b8f4a67c7b6 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -3292,6 +3292,12 @@ marker_priv_cleanup (xlator_t *this)
         LOCK_DESTROY (&priv->lock);
 
         GF_FREE (priv);
+
+        if (this->local_pool) {
+                mem_pool_destroy (this->local_pool);
+                this->local_pool = NULL;
+        }
+
 out:
         return;
 }
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index e8335e63c1c..d4ef8a5b2ce 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -5219,6 +5219,18 @@ out:
 void
 fini (xlator_t *this)
 {
+        quota_priv_t *priv = NULL;
+
+        priv = this->private;
+        if (!priv)
+                return;
+        this->private = NULL;
+        LOCK_DESTROY (&priv->lock);
+        GF_FREE (priv);
+        if (this->local_pool) {
+                mem_pool_destroy (this->local_pool);
+                this->local_pool = NULL;
+        }
         return;
 }
 
diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c
index cfdcc6ff5fc..e8f8b7bf051 100644
--- a/xlators/features/trash/src/trash.c
+++ b/xlators/features/trash/src/trash.c
@@ -33,7 +33,6 @@ trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          struct iatt *preoldparent, struct iatt *postoldparent,
                          struct iatt *prenewparent, struct iatt *postnewparent,
                          dict_t *xdata);
-
 /* Common routines used in this translator */
 
 /**
@@ -2431,6 +2430,7 @@ notify (xlator_t *this, int event, void *data, ...)
                         ret = create_internalop_directory (this);
 
         }
+
 out:
         ret = default_notify (this, event, data);
         if (ret)
@@ -2612,10 +2612,11 @@ void
 fini (xlator_t *this)
 {
         trash_private_t *priv = NULL;
+        inode_table_t     *inode_table = NULL;
 
         GF_VALIDATE_OR_GOTO ("trash", this, out);
         priv = this->private;
-
+        inode_table = priv->trash_itable;
         if (priv) {
                 if (priv->newtrash_dir)
                         GF_FREE (priv->newtrash_dir);
@@ -2625,9 +2626,17 @@ fini (xlator_t *this)
                         GF_FREE (priv->brick_path);
                 if (priv->eliminate)
                         wipe_eliminate_path (&priv->eliminate);
+                if (inode_table) {
+                        inode_table_destroy (inode_table);
+                        priv->trash_itable = NULL;
+                }
                 GF_FREE (priv);
         }
-        mem_pool_destroy (this->local_pool);
+
+        if (this->local_pool) {
+                mem_pool_destroy (this->local_pool);
+                this->local_pool = NULL;
+        }
         this->private = NULL;
 out:
         return;
diff --git a/xlators/features/upcall/src/upcall.c b/xlators/features/upcall/src/upcall.c
index b9b4bc9d725..d68bf5aa99a 100644
--- a/xlators/features/upcall/src/upcall.c
+++ b/xlators/features/upcall/src/upcall.c
@@ -2446,8 +2446,11 @@ fini (xlator_t *this)
 
         priv->fini = 1;
 
-        if (priv->reaper_init_done)
-                pthread_join (priv->reaper_thr, NULL);
+        if (priv->reaper_thr) {
+                gf_thread_cleanup_xint (priv->reaper_thr);
+                priv->reaper_thr = 0;
+                priv->reaper_init_done = _gf_false;
+        }
 
         dict_unref (priv->xattrs);
         LOCK_DESTROY (&priv->inode_ctx_lk);
@@ -2457,6 +2460,11 @@ fini (xlator_t *this)
          * before calling xlator_fini */
         GF_FREE (priv);
 
+        if (this->local_pool) {
+                mem_pool_destroy (this->local_pool);
+                this->local_pool = NULL;
+        }
+
         return 0;
 }
 
diff --git a/xlators/performance/decompounder/src/decompounder.c b/xlators/performance/decompounder/src/decompounder.c
index d3d9b9f7830..095a300e9f4 100644
--- a/xlators/performance/decompounder/src/decompounder.c
+++ b/xlators/performance/decompounder/src/decompounder.c
@@ -946,5 +946,12 @@ out:
 int32_t
 fini (xlator_t *this)
 {
+        if (!this)
+                return 0;
+
+        if (this->local_pool) {
+                mem_pool_destroy (this->local_pool);
+                this->local_pool = NULL;
+        }
         return 0;
 }
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index 501e4019e90..04d4760eea5 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -1136,8 +1136,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 {
         iot_conf_t *conf = this->private;
 
-        if ((GF_EVENT_PARENT_DOWN == event) ||
-            (GF_EVENT_CLEANUP == event))
+        if (GF_EVENT_PARENT_DOWN == event)
                 iot_exit_threads (conf);
 
         default_notify (this, event, data);
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 7bd276cdece..335c86fcda2 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -1500,7 +1500,6 @@ server_notify (xlator_t *this, int32_t event, void *data, ...)
                         glusterfs_mgmt_pmap_signout (ctx,
                                                      victim->name);
                         glusterfs_autoscale_threads (THIS->ctx, -1);
-                        default_notify (victim, GF_EVENT_CLEANUP, data);
 
                 }
                 break;
diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c
index fa2655ed8c0..a67b3d7dc70 100644
--- a/xlators/storage/posix/src/posix-common.c
+++ b/xlators/storage/posix/src/posix-common.c
@@ -105,6 +105,7 @@ extern char *marker_xattrs[];
         (lutimes (path, tv))
 #endif
 
+
 int32_t
 posix_priv (xlator_t *this)
 {
@@ -147,9 +148,6 @@ posix_notify (xlator_t *this,
         void *data,
         ...)
 {
-        struct posix_private *priv = NULL;
-
-        priv = this->private;
         switch (event)
         {
         case GF_EVENT_PARENT_UP:
@@ -158,31 +156,6 @@ posix_notify (xlator_t *this,
                 default_notify (this, GF_EVENT_CHILD_UP, data);
         }
         break;
-        case GF_EVENT_CLEANUP:
-                if (priv->health_check) {
-                        priv->health_check_active = _gf_false;
-                        pthread_cancel (priv->health_check);
-                        priv->health_check = 0;
-                }
-                if (priv->disk_space_check) {
-                        priv->disk_space_check_active = _gf_false;
-                        pthread_cancel (priv->disk_space_check);
-                        priv->disk_space_check = 0;
-                }
-                if (priv->janitor) {
-                        (void) gf_thread_cleanup_xint (priv->janitor);
-                        priv->janitor = 0;
-                }
-                if (priv->fsyncer) {
-                        (void) gf_thread_cleanup_xint (priv->fsyncer);
-                        priv->fsyncer = 0;
-                }
-                if (priv->mount_lock) {
-                        (void) sys_closedir (priv->mount_lock);
-                        priv->mount_lock = NULL;
-                }
-
-        break;
         default:
                 /* */
                 break;
@@ -1127,6 +1100,24 @@ posix_fini (xlator_t *this)
         if (!priv)
                 return;
         this->private = NULL;
+        if (priv->health_check) {
+                priv->health_check_active = _gf_false;
+                pthread_cancel (priv->health_check);
+                priv->health_check = 0;
+        }
+        if (priv->disk_space_check) {
+                priv->disk_space_check_active = _gf_false;
+                pthread_cancel (priv->disk_space_check);
+                priv->disk_space_check = 0;
+        }
+        if (priv->janitor) {
+                (void) gf_thread_cleanup_xint (priv->janitor);
+                priv->janitor = 0;
+        }
+        if (priv->fsyncer) {
+                (void) gf_thread_cleanup_xint (priv->fsyncer);
+                priv->fsyncer = 0;
+        }
         /*unlock brick dir*/
         if (priv->mount_lock)
                 (void) sys_closedir (priv->mount_lock);
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 56c9b4afe94..377389d22a3 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -2060,6 +2060,7 @@ abort:
                         gf_log (THIS->name, GF_LOG_INFO, "detaching not-only "
                                 " child %s", priv->base_path);
                         top->notify (top, GF_EVENT_CLEANUP, victim);
+                        xlator_mem_cleanup (victim);
                 }
         }