posix: use synctask for janitor

With brick mux, the number of threads increases as the number of
bricks increases. As an initiative to reduce the number of
threads in brick mux scenario, replacing janitor thread to use
synctask infra.

Now close() and closedir() handle by separate janitor
thread which is linked with glusterfs_ctx.

Updates #475
Change-Id: I0c4aaf728125ab7264442fde59f3d08542785f73
Signed-off-by: Poornima G <pgurusid@redhat.com>

7 files changed, 193 insertions, 79 deletions

diff --git a/xlators/storage/posix/src/Makefile.am b/xlators/storage/posix/src/Makefile.am
index d8af6221e4e..c080a229ff3 100644
--- a/xlators/storage/posix/src/Makefile.am
+++ b/xlators/storage/posix/src/Makefile.am
@@ -17,7 +17,7 @@ noinst_HEADERS = posix.h posix-mem-types.h posix-handle.h posix-aio.h \
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src
+	-I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/timer-wheel
 
 AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS) -I$(top_srcdir)/glusterfsd/src
 
diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c
index 111542ef3ae..a68a0dda938 100644
--- a/xlators/storage/posix/src/posix-common.c
+++ b/xlators/storage/posix/src/posix-common.c
@@ -60,6 +60,7 @@
 #include <glusterfs/events.h>
 #include "posix-gfid-path.h"
 #include <glusterfs/compat-uuid.h>
+#include "timer-wheel.h"
 
 extern char *marker_xattrs[];
 #define ALIGN_SIZE 4096
@@ -968,15 +969,12 @@ posix_init(xlator_t *this)
     if (_private->health_check_interval)
         posix_spawn_health_check_thread(this);
 
-    pthread_mutex_init(&_private->janitor_lock, NULL);
-    pthread_cond_init(&_private->janitor_cond, NULL);
-    INIT_LIST_HEAD(&_private->janitor_fds);
-
-    posix_spawn_janitor_thread(this);
+    posix_janitor_timer_start(this);
 
     pthread_mutex_init(&_private->fsync_mutex, NULL);
     pthread_cond_init(&_private->fsync_cond, NULL);
     INIT_LIST_HEAD(&_private->fsyncs);
+    posix_spawn_ctx_janitor_thread(this);
 
     ret = gf_thread_create(&_private->fsyncer, NULL, posix_fsyncer, this,
                            "posixfsy");
@@ -1072,6 +1070,7 @@ posix_fini(xlator_t *this)
 {
     struct posix_private *priv = this->private;
     gf_boolean_t health_check = _gf_false;
+    int ret = 0;
 
     if (!priv)
         return;
@@ -1093,8 +1092,13 @@ posix_fini(xlator_t *this)
         priv->disk_space_check = 0;
     }
     if (priv->janitor) {
-        (void)gf_thread_cleanup_xint(priv->janitor);
-        priv->janitor = 0;
+        /*TODO: Make sure the synctask is also complete */
+        ret = gf_tw_del_timer(this->ctx->tw->timer_wheel, priv->janitor);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_TIMER_DELETE_FAILED,
+                   "Failed to delete janitor timer");
+        }
+        priv->janitor = NULL;
     }
     if (priv->fsyncer) {
         (void)gf_thread_cleanup_xint(priv->fsyncer);
@@ -1106,7 +1110,6 @@ posix_fini(xlator_t *this)
 
     GF_FREE(priv->base_path);
     LOCK_DESTROY(&priv->lock);
-    pthread_mutex_destroy(&priv->janitor_lock);
     pthread_mutex_destroy(&priv->fsync_mutex);
     GF_FREE(priv->hostname);
     GF_FREE(priv->trash_path);
diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c
index ffd36d2c714..4d705586741 100644
--- a/xlators/storage/posix/src/posix-entry-ops.c
+++ b/xlators/storage/posix/src/posix-entry-ops.c
@@ -1366,7 +1366,6 @@ posix_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
                            priv->trash_path, gfid_str);
             gf_msg_debug(this->name, 0, "Moving %s to %s", real_path, tmp_path);
             op_ret = sys_rename(real_path, tmp_path);
-            pthread_cond_signal(&priv->janitor_cond);
         }
     } else {
         op_ret = sys_rmdir(real_path);
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 54fc1dc1195..208e319b336 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -57,6 +57,8 @@
 #include "posix-gfid-path.h"
 #include <glusterfs/events.h>
 #include "glusterfsd.h"
+#include "glusterfs/syncop.h"
+#include "timer-wheel.h"
 #include <sys/types.h>
 
 char *marker_xattrs[] = {"trusted.glusterfs.quota.*",
@@ -1409,81 +1411,181 @@ janitor_walker(const char *fpath, const struct stat *sb, int typeflag,
     return 0; /* 0 = FTW_CONTINUE */
 }
 
-static struct posix_fd *
-janitor_get_next_fd(xlator_t *this)
+void
+__posix_janitor_timer_start(xlator_t *this);
+
+static int
+posix_janitor_task_done(int ret, call_frame_t *frame, void *data)
 {
+    xlator_t *this = NULL;
     struct posix_private *priv = NULL;
-    struct posix_fd *pfd = NULL;
 
-    struct timespec timeout;
+    this = data;
+    priv = this->private;
+
+    LOCK(&priv->lock);
+    {
+        __posix_janitor_timer_start(this);
+    }
+    UNLOCK(&priv->lock);
+
+    return 0;
+}
+
+static int
+posix_janitor_task(void *data)
+{
+    xlator_t *this = NULL;
+    struct posix_private *priv = NULL;
+    xlator_t *old_this = NULL;
+
+    time_t now;
+
+    this = data;
+    priv = this->private;
+    /* We need THIS to be set for janitor_walker */
+    old_this = THIS;
+    THIS = this;
+
+    time(&now);
+    if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) {
+        if (priv->disable_landfill_purge) {
+            gf_msg_debug(this->name, 0,
+                         "Janitor would have "
+                         "cleaned out %s, but purge"
+                         "is disabled.",
+                         priv->trash_path);
+        } else {
+            gf_msg_trace(this->name, 0, "janitor cleaning out %s",
+                         priv->trash_path);
+
+            nftw(priv->trash_path, janitor_walker, 32, FTW_DEPTH | FTW_PHYS);
+        }
+        priv->last_landfill_check = now;
+    }
+
+    THIS = old_this;
+
+    return 0;
+}
+
+static void
+posix_janitor_task_initator(struct gf_tw_timer_list *timer, void *data,
+                            unsigned long calltime)
+{
+    xlator_t *this = NULL;
+    int ret = 0;
+
+    this = data;
+
+    ret = synctask_new(this->ctx->env, posix_janitor_task,
+                       posix_janitor_task_done, NULL, this);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED,
+               "spawning janitor "
+               "thread failed");
+    }
+
+    return;
+}
+
+void
+__posix_janitor_timer_start(xlator_t *this)
+{
+    struct posix_private *priv = NULL;
+    struct gf_tw_timer_list *timer = NULL;
+
+    priv = this->private;
+    timer = priv->janitor;
+
+    INIT_LIST_HEAD(&timer->entry);
+    timer->expires = priv->janitor_sleep_duration;
+    timer->function = posix_janitor_task_initator;
+    timer->data = this;
+    gf_tw_add_timer(this->ctx->tw->timer_wheel, timer);
+
+    return;
+}
+
+void
+posix_janitor_timer_start(xlator_t *this)
+{
+    struct posix_private *priv = NULL;
+    struct gf_tw_timer_list *timer = NULL;
 
     priv = this->private;
 
-    pthread_mutex_lock(&priv->janitor_lock);
+    LOCK(&priv->lock);
     {
-        if (list_empty(&priv->janitor_fds)) {
+        if (!priv->janitor) {
+            timer = GF_CALLOC(1, sizeof(struct gf_tw_timer_list),
+                              gf_common_mt_tw_timer_list);
+            if (!timer) {
+                goto unlock;
+            }
+            priv->janitor = timer;
+            __posix_janitor_timer_start(this);
+        }
+    }
+unlock:
+    UNLOCK(&priv->lock);
+
+    return;
+}
+
+static struct posix_fd *
+janitor_get_next_fd(glusterfs_ctx_t *ctx, int32_t janitor_sleep)
+{
+    struct posix_fd *pfd = NULL;
+
+    struct timespec timeout;
+
+    pthread_mutex_lock(&ctx->janitor_lock);
+    {
+        if (list_empty(&ctx->janitor_fds)) {
             time(&timeout.tv_sec);
-            timeout.tv_sec += priv->janitor_sleep_duration;
+            timeout.tv_sec += janitor_sleep;
             timeout.tv_nsec = 0;
 
-            pthread_cond_timedwait(&priv->janitor_cond, &priv->janitor_lock,
+            pthread_cond_timedwait(&ctx->janitor_cond, &ctx->janitor_lock,
                                    &timeout);
             goto unlock;
         }
 
-        pfd = list_entry(priv->janitor_fds.next, struct posix_fd, list);
+        pfd = list_entry(ctx->janitor_fds.next, struct posix_fd, list);
 
-        list_del(priv->janitor_fds.next);
+        list_del(ctx->janitor_fds.next);
     }
 unlock:
-    pthread_mutex_unlock(&priv->janitor_lock);
+    pthread_mutex_unlock(&ctx->janitor_lock);
 
     return pfd;
 }
 
 static void *
-posix_janitor_thread_proc(void *data)
+posix_ctx_janitor_thread_proc(void *data)
 {
     xlator_t *this = NULL;
-    struct posix_private *priv = NULL;
     struct posix_fd *pfd;
-
-    time_t now;
+    glusterfs_ctx_t *ctx = NULL;
+    struct posix_private *priv = NULL;
+    int32_t sleep_duration = 0;
 
     this = data;
-    priv = this->private;
-
+    ctx = THIS->ctx;
     THIS = this;
 
+    priv = this->private;
+    sleep_duration = priv->janitor_sleep_duration;
     while (1) {
-        time(&now);
-        if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) {
-            if (priv->disable_landfill_purge) {
-                gf_msg_debug(this->name, 0,
-                             "Janitor would have "
-                             "cleaned out %s, but purge"
-                             "is disabled.",
-                             priv->trash_path);
-            } else {
-                gf_msg_trace(this->name, 0, "janitor cleaning out %s",
-                             priv->trash_path);
-
-                nftw(priv->trash_path, janitor_walker, 32,
-                     FTW_DEPTH | FTW_PHYS);
-            }
-            priv->last_landfill_check = now;
-        }
-
-        pfd = janitor_get_next_fd(this);
+        pfd = janitor_get_next_fd(ctx, sleep_duration);
         if (pfd) {
             if (pfd->dir == NULL) {
                 gf_msg_trace(this->name, 0, "janitor: closing file fd=%d",
                              pfd->fd);
                 sys_close(pfd->fd);
             } else {
-                gf_msg_debug(this->name, 0,
-                             "janitor: closing"
-                             " dir fd=%p",
+                gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p",
                              pfd->dir);
                 sys_closedir(pfd->dir);
             }
@@ -1496,18 +1598,25 @@ posix_janitor_thread_proc(void *data)
 }
 
 void
-posix_spawn_janitor_thread(xlator_t *this)
+posix_spawn_ctx_janitor_thread(xlator_t *this)
 {
     struct posix_private *priv = NULL;
     int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
 
     priv = this->private;
+    ctx = THIS->ctx;
 
     LOCK(&priv->lock);
     {
-        if (!priv->janitor_present) {
-            ret = gf_thread_create(&priv->janitor, NULL,
-                                   posix_janitor_thread_proc, this, "posixjan");
+        if (!ctx->janitor) {
+            pthread_mutex_init(&ctx->janitor_lock, NULL);
+            pthread_cond_init(&ctx->janitor_cond, NULL);
+            INIT_LIST_HEAD(&ctx->janitor_fds);
+
+            ret = gf_thread_create(&ctx->janitor, NULL,
+                                   posix_ctx_janitor_thread_proc, this,
+                                   "posixctxjan");
 
             if (ret < 0) {
                 gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED,
@@ -1515,8 +1624,6 @@ posix_spawn_janitor_thread(xlator_t *this)
                        "thread failed");
                 goto unlock;
             }
-
-            priv->janitor_present = _gf_true;
         }
     }
 unlock:
diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c
index dafd1855ef9..d2324823592 100644
--- a/xlators/storage/posix/src/posix-inode-fd-ops.c
+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c
@@ -1236,8 +1236,7 @@ posix_releasedir(xlator_t *this, fd_t *fd)
     struct posix_fd *pfd = NULL;
     uint64_t tmp_pfd = 0;
     int ret = 0;
-
-    struct posix_private *priv = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
     VALIDATE_OR_GOTO(this, out);
     VALIDATE_OR_GOTO(fd, out);
@@ -1255,18 +1254,21 @@ posix_releasedir(xlator_t *this, fd_t *fd)
         goto out;
     }
 
-    priv = this->private;
-    if (!priv)
-        goto out;
+    ctx = THIS->ctx;
 
-    pthread_mutex_lock(&priv->janitor_lock);
+    pthread_mutex_lock(&ctx->janitor_lock);
     {
         INIT_LIST_HEAD(&pfd->list);
-        list_add_tail(&pfd->list, &priv->janitor_fds);
-        pthread_cond_signal(&priv->janitor_cond);
+        list_add_tail(&pfd->list, &ctx->janitor_fds);
+        pthread_cond_signal(&ctx->janitor_cond);
     }
-    pthread_mutex_unlock(&priv->janitor_lock);
+    pthread_mutex_unlock(&ctx->janitor_lock);
+
+    /*gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir);
 
+    sys_closedir(pfd->dir);
+    GF_FREE(pfd);
+    */
 out:
     return 0;
 }
@@ -2337,11 +2339,13 @@ posix_release(xlator_t *this, fd_t *fd)
     struct posix_fd *pfd = NULL;
     int ret = -1;
     uint64_t tmp_pfd = 0;
+    glusterfs_ctx_t *ctx = NULL;
 
     VALIDATE_OR_GOTO(this, out);
     VALIDATE_OR_GOTO(fd, out);
 
     priv = this->private;
+    ctx = THIS->ctx;
 
     ret = fd_ctx_del(fd, this, &tmp_pfd);
     if (ret < 0) {
@@ -2349,22 +2353,23 @@ posix_release(xlator_t *this, fd_t *fd)
                "pfd is NULL from fd=%p", fd);
         goto out;
     }
-    pfd = (struct posix_fd *)(long)tmp_pfd;
 
+    pfd = (struct posix_fd *)(long)tmp_pfd;
     if (pfd->dir) {
         gf_msg(this->name, GF_LOG_WARNING, 0, P_MSG_DIR_NOT_NULL,
                "pfd->dir is %p (not NULL) for file fd=%p", pfd->dir, fd);
     }
-    if (!priv)
-        goto out;
 
-    pthread_mutex_lock(&priv->janitor_lock);
+    pthread_mutex_lock(&ctx->janitor_lock);
     {
         INIT_LIST_HEAD(&pfd->list);
-        list_add_tail(&pfd->list, &priv->janitor_fds);
-        pthread_cond_signal(&priv->janitor_cond);
+        list_add_tail(&pfd->list, &ctx->janitor_fds);
+        pthread_cond_signal(&ctx->janitor_cond);
     }
-    pthread_mutex_unlock(&priv->janitor_lock);
+    pthread_mutex_unlock(&ctx->janitor_lock);
+
+    if (!priv)
+        goto out;
 
     LOCK(&priv->lock);
     {
diff --git a/xlators/storage/posix/src/posix-messages.h b/xlators/storage/posix/src/posix-messages.h
index 928723db8f9..32292756bc4 100644
--- a/xlators/storage/posix/src/posix-messages.h
+++ b/xlators/storage/posix/src/posix-messages.h
@@ -68,6 +68,6 @@ GLFS_MSGID(POSIX, P_MSG_XATTR_FAILED, P_MSG_NULL_GFID, P_MSG_FCNTL_FAILED,
            P_MSG_FALLOCATE_FAILED, P_MSG_STOREMDATA_FAILED,
            P_MSG_FETCHMDATA_FAILED, P_MSG_GETMDATA_FAILED,
            P_MSG_SETMDATA_FAILED, P_MSG_FRESHFILE, P_MSG_MUTEX_FAILED,
-           P_MSG_COPY_FILE_RANGE_FAILED);
+           P_MSG_COPY_FILE_RANGE_FAILED, P_MSG_TIMER_DELETE_FAILED);
 
 #endif /* !_GLUSTERD_MESSAGES_H_ */
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index a1ec996f4b2..97c9666ab44 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -151,9 +151,6 @@ struct posix_private {
 
     time_t last_landfill_check;
     int32_t janitor_sleep_duration;
-    struct list_head janitor_fds;
-    pthread_cond_t janitor_cond;
-    pthread_mutex_t janitor_lock;
 
     int64_t read_value;  /* Total read, from init */
     int64_t write_value; /* Total write, from init */
@@ -179,9 +176,9 @@ struct posix_private {
     */
     gf_boolean_t background_unlink;
 
-    /* janitor thread which cleans up /.trash (created by replicate) */
-    pthread_t janitor;
-    gf_boolean_t janitor_present;
+    /* janitor task which cleans up /.trash (created by replicate) */
+    struct gf_tw_timer_list *janitor;
+
     char *trash_path;
     /* lock for brick dir */
     DIR *mount_lock;
@@ -344,7 +341,7 @@ int
 posix_fhandle_pair(call_frame_t *frame, xlator_t *this, int fd, char *key,
                    data_t *value, int flags, struct iatt *stbuf, fd_t *_fd);
 void
-posix_spawn_janitor_thread(xlator_t *this);
+posix_janitor_timer_start(xlator_t *this);
 int
 posix_acl_xattr_set(xlator_t *this, const char *path, dict_t *xattr_req);
 int
@@ -664,4 +661,7 @@ posix_cs_maintenance(xlator_t *this, fd_t *fd, loc_t *loc, int *pfd,
 int
 posix_check_dev_file(xlator_t *this, inode_t *inode, char *fop, int *op_errno);
 
+void
+posix_spawn_ctx_janitor_thread(xlator_t *this);
+
 #endif /* _POSIX_H */