Posix: Use simple approach to close fd

Problem: posix_release(dir) functions add the fd's into a ctx->janitor_fds
         and janitor thread closes the fd's.In brick_mux environment it is
         difficult to handle race condition in janitor threads because brick
         spawns a single janitor thread for all bricks.

Solution: Use synctask to execute posix_release(dir) functions instead of 
          using background a thread to close fds.

Credits: Pranith Karampuri <pkarampu@redhat.com>
Change-Id: Iffb031f0695a7da83d5a2f6bac8863dad225317e
Fixes: bz#1811631
Signed-off-by: Mohit Agrawal <moagrawal@redhat.com>

4 files changed, 6 insertions, 127 deletions

diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c
index f4003942f4e..20a8d231181 100644
--- a/xlators/storage/posix/src/posix-common.c
+++ b/xlators/storage/posix/src/posix-common.c
@@ -1029,10 +1029,6 @@ posix_init(xlator_t *this)
     pthread_mutex_init(&_private->janitor_mutex, NULL);
     pthread_cond_init(&_private->janitor_cond, NULL);
     INIT_LIST_HEAD(&_private->fsyncs);
-    ret = posix_spawn_ctx_janitor_thread(this);
-    if (ret)
-        goto out;
-
     ret = gf_thread_create(&_private->fsyncer, NULL, posix_fsyncer, this,
                            "posixfsy");
     if (ret) {
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 4919cbdf2e9..d18db1098fc 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1587,104 +1587,6 @@ unlock:
     return;
 }
 
-static struct posix_fd *
-janitor_get_next_fd(glusterfs_ctx_t *ctx, int32_t janitor_sleep)
-{
-    struct posix_fd *pfd = NULL;
-
-    struct timespec timeout;
-
-    pthread_mutex_lock(&ctx->janitor_lock);
-    {
-        if (list_empty(&ctx->janitor_fds)) {
-            time(&timeout.tv_sec);
-            timeout.tv_sec += janitor_sleep;
-            timeout.tv_nsec = 0;
-
-            pthread_cond_timedwait(&ctx->janitor_cond, &ctx->janitor_lock,
-                                   &timeout);
-            goto unlock;
-        }
-
-        pfd = list_entry(ctx->janitor_fds.next, struct posix_fd, list);
-
-        list_del(ctx->janitor_fds.next);
-    }
-unlock:
-    pthread_mutex_unlock(&ctx->janitor_lock);
-
-    return pfd;
-}
-
-static void *
-posix_ctx_janitor_thread_proc(void *data)
-{
-    xlator_t *this = NULL;
-    struct posix_fd *pfd;
-    glusterfs_ctx_t *ctx = NULL;
-    struct posix_private *priv = NULL;
-    int32_t sleep_duration = 0;
-
-    this = data;
-    ctx = THIS->ctx;
-    THIS = this;
-
-    priv = this->private;
-    sleep_duration = priv->janitor_sleep_duration;
-    while (1) {
-        pfd = janitor_get_next_fd(ctx, sleep_duration);
-        if (pfd) {
-            if (pfd->dir == NULL) {
-                gf_msg_trace(this->name, 0, "janitor: closing file fd=%d",
-                             pfd->fd);
-                sys_close(pfd->fd);
-            } else {
-                gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p",
-                             pfd->dir);
-                sys_closedir(pfd->dir);
-            }
-
-            GF_FREE(pfd);
-        }
-    }
-
-    return NULL;
-}
-
-int
-posix_spawn_ctx_janitor_thread(xlator_t *this)
-{
-    struct posix_private *priv = NULL;
-    int ret = 0;
-    glusterfs_ctx_t *ctx = NULL;
-
-    priv = this->private;
-    ctx = THIS->ctx;
-
-    LOCK(&priv->lock);
-    {
-        if (!ctx->janitor) {
-            pthread_mutex_init(&ctx->janitor_lock, NULL);
-            pthread_cond_init(&ctx->janitor_cond, NULL);
-            INIT_LIST_HEAD(&ctx->janitor_fds);
-
-            ret = gf_thread_create(&ctx->janitor, NULL,
-                                   posix_ctx_janitor_thread_proc, this,
-                                   "posixctxjan");
-
-            if (ret) {
-                gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED,
-                       "spawning janitor "
-                       "thread failed");
-                goto unlock;
-            }
-        }
-    }
-unlock:
-    UNLOCK(&priv->lock);
-    return ret;
-}
-
 static int
 is_fresh_file(int64_t sec, int64_t ns)
 {
diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c
index a6c2b512ef1..439a4362fc6 100644
--- a/xlators/storage/posix/src/posix-inode-fd-ops.c
+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c
@@ -1366,7 +1366,6 @@ posix_releasedir(xlator_t *this, fd_t *fd)
     struct posix_fd *pfd = NULL;
     uint64_t tmp_pfd = 0;
     int ret = 0;
-    glusterfs_ctx_t *ctx = NULL;
 
     VALIDATE_OR_GOTO(this, out);
     VALIDATE_OR_GOTO(fd, out);
@@ -1384,21 +1383,11 @@ posix_releasedir(xlator_t *this, fd_t *fd)
         goto out;
     }
 
-    ctx = THIS->ctx;
-
-    pthread_mutex_lock(&ctx->janitor_lock);
-    {
-        INIT_LIST_HEAD(&pfd->list);
-        list_add_tail(&pfd->list, &ctx->janitor_fds);
-        pthread_cond_signal(&ctx->janitor_cond);
-    }
-    pthread_mutex_unlock(&ctx->janitor_lock);
-
-    /*gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir);
+    gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir);
 
     sys_closedir(pfd->dir);
     GF_FREE(pfd);
-    */
+
 out:
     return 0;
 }
@@ -2525,13 +2514,11 @@ posix_release(xlator_t *this, fd_t *fd)
     struct posix_fd *pfd = NULL;
     int ret = -1;
     uint64_t tmp_pfd = 0;
-    glusterfs_ctx_t *ctx = NULL;
 
     VALIDATE_OR_GOTO(this, out);
     VALIDATE_OR_GOTO(fd, out);
 
     priv = this->private;
-    ctx = THIS->ctx;
 
     ret = fd_ctx_del(fd, this, &tmp_pfd);
     if (ret < 0) {
@@ -2546,13 +2533,10 @@ posix_release(xlator_t *this, fd_t *fd)
                "pfd->dir is %p (not NULL) for file fd=%p", pfd->dir, fd);
     }
 
-    pthread_mutex_lock(&ctx->janitor_lock);
-    {
-        INIT_LIST_HEAD(&pfd->list);
-        list_add_tail(&pfd->list, &ctx->janitor_fds);
-        pthread_cond_signal(&ctx->janitor_cond);
-    }
-    pthread_mutex_unlock(&ctx->janitor_lock);
+    gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir);
+
+    sys_close(pfd->fd);
+    GF_FREE(pfd);
 
     if (!priv)
         goto out;
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index 359b838f34b..fb609dff731 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -659,9 +659,6 @@ posix_cs_maintenance(xlator_t *this, fd_t *fd, loc_t *loc, int *pfd,
 int
 posix_check_dev_file(xlator_t *this, inode_t *inode, char *fop, int *op_errno);
 
-int
-posix_spawn_ctx_janitor_thread(xlator_t *this);
-
 void
 posix_update_iatt_buf(struct iatt *buf, int fd, char *loc, dict_t *xdata);