core: make the per glusterfs_ctx_t timer-wheel refcounted

xlators can use a 'global' timer-wheel for scheduling events. This
timer-wheel is managed per glusterfs_ctx_t, but does not need to be
allocated for every graph. When an xlator wants to use the timer-wheel,
it will be instanciated on demand, and provided to xlators that request
it later on.

By adding a reference counter to the glusterfs_ctx_t for the
timer-wheel, the threads and structures can be cleaned up when the last
xlator does not have a need for it anymore. In general, the xlators
request the timer-wheel in init(), and they should return it in fini().

Because the timer-wheel is managed per glusterfs_ctx_t, the functions
can be added to ctx.c and do not need to live in their very minimal
tw.[ch] files.

Change-Id: I19d225b39aaa272d9005ba7adc3104c3764f1572
BUG: 1442788
Reported-by: Poornima G <pgurusid@redhat.com>
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: https://review.gluster.org/17068
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Zhou Zhengping <johnzzpcrystal@gmail.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

7 files changed, 56 insertions, 57 deletions

diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index 2311c53645a..52b73166ddb 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -32,7 +32,7 @@ libglusterfs_la_SOURCES = dict.c xlator.c logging.c \
 	strfd.c parse-utils.c $(CONTRIBDIR)/mount/mntent.c \
 	$(CONTRIBDIR)/libexecinfo/execinfo.c quota-common-utils.c rot-buffs.c \
 	$(CONTRIBDIR)/timer-wheel/timer-wheel.c \
-	$(CONTRIBDIR)/timer-wheel/find_last_bit.c tw.c default-args.c locking.c \
+	$(CONTRIBDIR)/timer-wheel/find_last_bit.c default-args.c locking.c \
 	compound-fop-utils.c throttle-tbf.c
 
 nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c defaults.c
@@ -50,7 +50,7 @@ libglusterfs_la_HEADERS = common-utils.h defaults.h default-args.h \
 	refcount.h run.h options.h lkowner.h fd-lk.h circ-buff.h \
 	event-history.h gidcache.h client_t.h glusterfs-acl.h \
 	glfs-message-id.h template-component-messages.h strfd.h \
-	syncop-utils.h parse-utils.h libglusterfs-messages.h tw.h \
+	syncop-utils.h parse-utils.h libglusterfs-messages.h \
 	lvm-defaults.h quota-common-utils.h rot-buffs.h \
 	compat-uuid.h upcall-utils.h throttle-tbf.h events.h\
 	compound-fop-utils.h atomic.h
diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c
index d3fb39822d1..1c707eb5dfd 100644
--- a/libglusterfs/src/ctx.c
+++ b/libglusterfs/src/ctx.c
@@ -9,9 +9,10 @@
 */
 
 #include <pthread.h>
-#include "globals.h"
 
+#include "globals.h"
 #include "glusterfs.h"
+#include "timer-wheel.h"
 
 glusterfs_ctx_t *
 glusterfs_ctx_new ()
@@ -52,3 +53,39 @@ out:
 	return ctx;
 }
 
+static void
+glusterfs_ctx_tw_destroy (struct gf_ctx_tw *ctx_tw)
+{
+        if (ctx_tw->timer_wheel)
+                gf_tw_cleanup_timers (ctx_tw->timer_wheel);
+
+        GF_FREE (ctx_tw);
+}
+
+struct tvec_base*
+glusterfs_ctx_tw_get (glusterfs_ctx_t *ctx)
+{
+        struct gf_ctx_tw *ctx_tw = NULL;
+
+        LOCK (&ctx->lock);
+        {
+                if (ctx->tw) {
+                        ctx_tw = GF_REF_GET (ctx->tw);
+                } else {
+                        ctx_tw = GF_CALLOC (1, sizeof (struct gf_ctx_tw),
+                                            gf_common_mt_tw_ctx);
+                        ctx_tw->timer_wheel = gf_tw_init_timers();
+                        GF_REF_INIT (ctx_tw, glusterfs_ctx_tw_destroy);
+                        ctx->tw = ctx_tw;
+                }
+        }
+        UNLOCK (&ctx->lock);
+
+        return ctx_tw->timer_wheel;
+}
+
+void
+glusterfs_ctx_tw_put (glusterfs_ctx_t *ctx)
+{
+        GF_REF_PUT (ctx->tw);
+}
diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 5f4a938a08f..4c668dcb19c 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -116,6 +116,9 @@ char *glusterfs_leaseid_buf_get (void);
 /* init */
 int glusterfs_globals_init (glusterfs_ctx_t *ctx);
 
+struct tvec_base* glusterfs_ctx_tw_get (glusterfs_ctx_t *ctx);
+void glusterfs_ctx_tw_put (glusterfs_ctx_t *ctx);
+
 extern const char *gf_fop_list[];
 extern const char *gf_upcall_list[];
 
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 4d5ca839cd3..839a1d47d2a 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -36,6 +36,7 @@
 #include "logging.h"
 #include "lkowner.h"
 #include "compat-uuid.h"
+#include "refcount.h"
 
 #define GF_YES 1
 #define GF_NO  0
@@ -438,6 +439,12 @@ typedef enum {
 
 struct tvec_base;
 
+/* reference counting for the global (per ctx) timer-wheel */
+struct gf_ctx_tw {
+        GF_REF_DECL;
+        struct tvec_base *timer_wheel; /* global timer-wheel instance */
+};
+
 struct _glusterfs_ctx {
         cmd_args_t          cmd_args;
         char               *process_uuid;
@@ -506,14 +513,13 @@ struct _glusterfs_ctx {
          */
         mgmt_ssl_t          secure_srvr;
         /* Buffer to 'save' backtrace even under OOM-kill like situations*/
-        char btbuf[GF_BACKTRACE_LEN];
+        char                btbuf[GF_BACKTRACE_LEN];
 
-        pthread_mutex_t notify_lock;
-        pthread_cond_t notify_cond;
-        int notifying;
-
-        struct tvec_base *timer_wheel; /* global timer-wheel instance */
+        pthread_mutex_t     notify_lock;
+        pthread_cond_t      notify_cond;
+        int                 notifying;
 
+        struct gf_ctx_tw   *tw; /* refcounted timer_wheel */
 };
 typedef struct _glusterfs_ctx glusterfs_ctx_t;
 
diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h
index 6c0472f00b8..59c7e77d548 100644
--- a/libglusterfs/src/mem-types.h
+++ b/libglusterfs/src/mem-types.h
@@ -164,6 +164,7 @@ enum gf_common_mem_types_ {
         /*used for compound fops*/
         gf_mt_compound_req_t,
         gf_mt_compound_rsp_t,
+        gf_common_mt_tw_ctx,
         gf_common_mt_tw_timer_list,
         /*lock migration*/
         gf_common_mt_lock_mig,
diff --git a/libglusterfs/src/tw.c b/libglusterfs/src/tw.c
deleted file mode 100644
index fa11998aace..00000000000
--- a/libglusterfs/src/tw.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
-  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "tw.h"
-#include "timer-wheel.h"
-
-int
-glusterfs_global_timer_wheel_init (glusterfs_ctx_t *ctx)
-{
-        ctx->timer_wheel = gf_tw_init_timers();
-        return ctx->timer_wheel ? 0 : -1;
-}
-
-struct tvec_base *
-glusterfs_global_timer_wheel (xlator_t *this)
-{
-        return this->ctx->timer_wheel;
-}
diff --git a/libglusterfs/src/tw.h b/libglusterfs/src/tw.h
deleted file mode 100644
index e635cd2b496..00000000000
--- a/libglusterfs/src/tw.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
-  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __TW_H__
-#define __TW_H__
-
-#include "xlator.h"
-#include "glusterfs.h"
-
-int
-glusterfs_global_timer_wheel_init (glusterfs_ctx_t *);
-
-struct tvec_base *
-glusterfs_global_timer_wheel (xlator_t *);
-
-#endif /* __TW_H__ */