summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPoornima G <pgurusid@redhat.com>2018-08-16 16:36:08 +0530
committerAmar Tumballi <amarts@redhat.com>2018-12-19 14:36:52 +0000
commit2af8fca492f3e1152177641b5e6ab9ea59ec0acd (patch)
tree65dfe1889f015c72f8dcad045bebc2a2197102d0
parente3ec41af9a9f4d906dd7b512b3f4f91a6f338f4b (diff)
posix: use synctask for janitor
With brick mux, the number of threads increases as the number of bricks increases. As an initiative to reduce the number of threads in brick mux scenario, replacing janitor thread to use synctask infra. Now close() and closedir() handle by separate janitor thread which is linked with glusterfs_ctx. Updates #475 Change-Id: I0c4aaf728125ab7264442fde59f3d08542785f73 Signed-off-by: Poornima G <pgurusid@redhat.com>
-rw-r--r--glusterfsd/src/glusterfsd.c8
-rw-r--r--libglusterfs/src/glusterfs/glusterfs.h6
-rw-r--r--xlators/features/leases/src/leases.c6
-rw-r--r--xlators/storage/posix/src/Makefile.am2
-rw-r--r--xlators/storage/posix/src/posix-common.c19
-rw-r--r--xlators/storage/posix/src/posix-entry-ops.c1
-rw-r--r--xlators/storage/posix/src/posix-helpers.c197
-rw-r--r--xlators/storage/posix/src/posix-inode-fd-ops.c37
-rw-r--r--xlators/storage/posix/src/posix-messages.h2
-rw-r--r--xlators/storage/posix/src/posix.h14
10 files changed, 205 insertions, 87 deletions
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 0dea52b6af1..7753c629216 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -2837,11 +2837,9 @@ main(int argc, char *argv[])
}
/* do this _after_ daemonize() */
- if (cmd->global_timer_wheel) {
- if (!glusterfs_ctx_tw_get(ctx)) {
- ret = -1;
- goto out;
- }
+ if (!glusterfs_ctx_tw_get(ctx)) {
+ ret = -1;
+ goto out;
}
ret = glusterfs_volumes_init(ctx);
diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h
index 325241818c0..ae137c5d334 100644
--- a/libglusterfs/src/glusterfs/glusterfs.h
+++ b/libglusterfs/src/glusterfs/glusterfs.h
@@ -716,6 +716,12 @@ struct _glusterfs_ctx {
} stats;
struct list_head volfile_list;
+
+ /* Add members to manage janitor threads for cleanup fd */
+ struct list_head janitor_fds;
+ pthread_cond_t janitor_cond;
+ pthread_mutex_t janitor_lock;
+ pthread_t janitor;
};
typedef struct _glusterfs_ctx glusterfs_ctx_t;
diff --git a/xlators/features/leases/src/leases.c b/xlators/features/leases/src/leases.c
index de74312b0ba..56299632e1d 100644
--- a/xlators/features/leases/src/leases.c
+++ b/xlators/features/leases/src/leases.c
@@ -1019,11 +1019,11 @@ fini(xlator_t *this)
priv->inited_recall_thr = _gf_false;
}
- GF_FREE(priv);
- if (this->ctx->tw) {
+ if (priv->timer_wheel) {
glusterfs_ctx_tw_put(this->ctx);
- this->ctx->tw = NULL;
}
+
+ GF_FREE(priv);
return;
}
diff --git a/xlators/storage/posix/src/Makefile.am b/xlators/storage/posix/src/Makefile.am
index d8af6221e4e..c080a229ff3 100644
--- a/xlators/storage/posix/src/Makefile.am
+++ b/xlators/storage/posix/src/Makefile.am
@@ -17,7 +17,7 @@ noinst_HEADERS = posix.h posix-mem-types.h posix-handle.h posix-aio.h \
AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
- -I$(top_srcdir)/rpc/rpc-lib/src
+ -I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/timer-wheel
AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS) -I$(top_srcdir)/glusterfsd/src
diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c
index 111542ef3ae..a68a0dda938 100644
--- a/xlators/storage/posix/src/posix-common.c
+++ b/xlators/storage/posix/src/posix-common.c
@@ -60,6 +60,7 @@
#include <glusterfs/events.h>
#include "posix-gfid-path.h"
#include <glusterfs/compat-uuid.h>
+#include "timer-wheel.h"
extern char *marker_xattrs[];
#define ALIGN_SIZE 4096
@@ -968,15 +969,12 @@ posix_init(xlator_t *this)
if (_private->health_check_interval)
posix_spawn_health_check_thread(this);
- pthread_mutex_init(&_private->janitor_lock, NULL);
- pthread_cond_init(&_private->janitor_cond, NULL);
- INIT_LIST_HEAD(&_private->janitor_fds);
-
- posix_spawn_janitor_thread(this);
+ posix_janitor_timer_start(this);
pthread_mutex_init(&_private->fsync_mutex, NULL);
pthread_cond_init(&_private->fsync_cond, NULL);
INIT_LIST_HEAD(&_private->fsyncs);
+ posix_spawn_ctx_janitor_thread(this);
ret = gf_thread_create(&_private->fsyncer, NULL, posix_fsyncer, this,
"posixfsy");
@@ -1072,6 +1070,7 @@ posix_fini(xlator_t *this)
{
struct posix_private *priv = this->private;
gf_boolean_t health_check = _gf_false;
+ int ret = 0;
if (!priv)
return;
@@ -1093,8 +1092,13 @@ posix_fini(xlator_t *this)
priv->disk_space_check = 0;
}
if (priv->janitor) {
- (void)gf_thread_cleanup_xint(priv->janitor);
- priv->janitor = 0;
+ /*TODO: Make sure the synctask is also complete */
+ ret = gf_tw_del_timer(this->ctx->tw->timer_wheel, priv->janitor);
+ if (ret < 0) {
+ gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_TIMER_DELETE_FAILED,
+ "Failed to delete janitor timer");
+ }
+ priv->janitor = NULL;
}
if (priv->fsyncer) {
(void)gf_thread_cleanup_xint(priv->fsyncer);
@@ -1106,7 +1110,6 @@ posix_fini(xlator_t *this)
GF_FREE(priv->base_path);
LOCK_DESTROY(&priv->lock);
- pthread_mutex_destroy(&priv->janitor_lock);
pthread_mutex_destroy(&priv->fsync_mutex);
GF_FREE(priv->hostname);
GF_FREE(priv->trash_path);
diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c
index ffd36d2c714..4d705586741 100644
--- a/xlators/storage/posix/src/posix-entry-ops.c
+++ b/xlators/storage/posix/src/posix-entry-ops.c
@@ -1366,7 +1366,6 @@ posix_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
priv->trash_path, gfid_str);
gf_msg_debug(this->name, 0, "Moving %s to %s", real_path, tmp_path);
op_ret = sys_rename(real_path, tmp_path);
- pthread_cond_signal(&priv->janitor_cond);
}
} else {
op_ret = sys_rmdir(real_path);
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 54fc1dc1195..208e319b336 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -57,6 +57,8 @@
#include "posix-gfid-path.h"
#include <glusterfs/events.h>
#include "glusterfsd.h"
+#include "glusterfs/syncop.h"
+#include "timer-wheel.h"
#include <sys/types.h>
char *marker_xattrs[] = {"trusted.glusterfs.quota.*",
@@ -1409,81 +1411,181 @@ janitor_walker(const char *fpath, const struct stat *sb, int typeflag,
return 0; /* 0 = FTW_CONTINUE */
}
-static struct posix_fd *
-janitor_get_next_fd(xlator_t *this)
+void
+__posix_janitor_timer_start(xlator_t *this);
+
+static int
+posix_janitor_task_done(int ret, call_frame_t *frame, void *data)
{
+ xlator_t *this = NULL;
struct posix_private *priv = NULL;
- struct posix_fd *pfd = NULL;
- struct timespec timeout;
+ this = data;
+ priv = this->private;
+
+ LOCK(&priv->lock);
+ {
+ __posix_janitor_timer_start(this);
+ }
+ UNLOCK(&priv->lock);
+
+ return 0;
+}
+
+static int
+posix_janitor_task(void *data)
+{
+ xlator_t *this = NULL;
+ struct posix_private *priv = NULL;
+ xlator_t *old_this = NULL;
+
+ time_t now;
+
+ this = data;
+ priv = this->private;
+ /* We need THIS to be set for janitor_walker */
+ old_this = THIS;
+ THIS = this;
+
+ time(&now);
+ if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) {
+ if (priv->disable_landfill_purge) {
+ gf_msg_debug(this->name, 0,
+ "Janitor would have "
+ "cleaned out %s, but purge"
+ "is disabled.",
+ priv->trash_path);
+ } else {
+ gf_msg_trace(this->name, 0, "janitor cleaning out %s",
+ priv->trash_path);
+
+ nftw(priv->trash_path, janitor_walker, 32, FTW_DEPTH | FTW_PHYS);
+ }
+ priv->last_landfill_check = now;
+ }
+
+ THIS = old_this;
+
+ return 0;
+}
+
+static void
+posix_janitor_task_initator(struct gf_tw_timer_list *timer, void *data,
+ unsigned long calltime)
+{
+ xlator_t *this = NULL;
+ int ret = 0;
+
+ this = data;
+
+ ret = synctask_new(this->ctx->env, posix_janitor_task,
+ posix_janitor_task_done, NULL, this);
+ if (ret < 0) {
+ gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED,
+ "spawning janitor "
+ "thread failed");
+ }
+
+ return;
+}
+
+void
+__posix_janitor_timer_start(xlator_t *this)
+{
+ struct posix_private *priv = NULL;
+ struct gf_tw_timer_list *timer = NULL;
+
+ priv = this->private;
+ timer = priv->janitor;
+
+ INIT_LIST_HEAD(&timer->entry);
+ timer->expires = priv->janitor_sleep_duration;
+ timer->function = posix_janitor_task_initator;
+ timer->data = this;
+ gf_tw_add_timer(this->ctx->tw->timer_wheel, timer);
+
+ return;
+}
+
+void
+posix_janitor_timer_start(xlator_t *this)
+{
+ struct posix_private *priv = NULL;
+ struct gf_tw_timer_list *timer = NULL;
priv = this->private;
- pthread_mutex_lock(&priv->janitor_lock);
+ LOCK(&priv->lock);
{
- if (list_empty(&priv->janitor_fds)) {
+ if (!priv->janitor) {
+ timer = GF_CALLOC(1, sizeof(struct gf_tw_timer_list),
+ gf_common_mt_tw_timer_list);
+ if (!timer) {
+ goto unlock;
+ }
+ priv->janitor = timer;
+ __posix_janitor_timer_start(this);
+ }
+ }
+unlock:
+ UNLOCK(&priv->lock);
+
+ return;
+}
+
+static struct posix_fd *
+janitor_get_next_fd(glusterfs_ctx_t *ctx, int32_t janitor_sleep)
+{
+ struct posix_fd *pfd = NULL;
+
+ struct timespec timeout;
+
+ pthread_mutex_lock(&ctx->janitor_lock);
+ {
+ if (list_empty(&ctx->janitor_fds)) {
time(&timeout.tv_sec);
- timeout.tv_sec += priv->janitor_sleep_duration;
+ timeout.tv_sec += janitor_sleep;
timeout.tv_nsec = 0;
- pthread_cond_timedwait(&priv->janitor_cond, &priv->janitor_lock,
+ pthread_cond_timedwait(&ctx->janitor_cond, &ctx->janitor_lock,
&timeout);
goto unlock;
}
- pfd = list_entry(priv->janitor_fds.next, struct posix_fd, list);
+ pfd = list_entry(ctx->janitor_fds.next, struct posix_fd, list);
- list_del(priv->janitor_fds.next);
+ list_del(ctx->janitor_fds.next);
}
unlock:
- pthread_mutex_unlock(&priv->janitor_lock);
+ pthread_mutex_unlock(&ctx->janitor_lock);
return pfd;
}
static void *
-posix_janitor_thread_proc(void *data)
+posix_ctx_janitor_thread_proc(void *data)
{
xlator_t *this = NULL;
- struct posix_private *priv = NULL;
struct posix_fd *pfd;
-
- time_t now;
+ glusterfs_ctx_t *ctx = NULL;
+ struct posix_private *priv = NULL;
+ int32_t sleep_duration = 0;
this = data;
- priv = this->private;
-
+ ctx = THIS->ctx;
THIS = this;
+ priv = this->private;
+ sleep_duration = priv->janitor_sleep_duration;
while (1) {
- time(&now);
- if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) {
- if (priv->disable_landfill_purge) {
- gf_msg_debug(this->name, 0,
- "Janitor would have "
- "cleaned out %s, but purge"
- "is disabled.",
- priv->trash_path);
- } else {
- gf_msg_trace(this->name, 0, "janitor cleaning out %s",
- priv->trash_path);
-
- nftw(priv->trash_path, janitor_walker, 32,
- FTW_DEPTH | FTW_PHYS);
- }
- priv->last_landfill_check = now;
- }
-
- pfd = janitor_get_next_fd(this);
+ pfd = janitor_get_next_fd(ctx, sleep_duration);
if (pfd) {
if (pfd->dir == NULL) {
gf_msg_trace(this->name, 0, "janitor: closing file fd=%d",
pfd->fd);
sys_close(pfd->fd);
} else {
- gf_msg_debug(this->name, 0,
- "janitor: closing"
- " dir fd=%p",
+ gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p",
pfd->dir);
sys_closedir(pfd->dir);
}
@@ -1496,18 +1598,25 @@ posix_janitor_thread_proc(void *data)
}
void
-posix_spawn_janitor_thread(xlator_t *this)
+posix_spawn_ctx_janitor_thread(xlator_t *this)
{
struct posix_private *priv = NULL;
int ret = 0;
+ glusterfs_ctx_t *ctx = NULL;
priv = this->private;
+ ctx = THIS->ctx;
LOCK(&priv->lock);
{
- if (!priv->janitor_present) {
- ret = gf_thread_create(&priv->janitor, NULL,
- posix_janitor_thread_proc, this, "posixjan");
+ if (!ctx->janitor) {
+ pthread_mutex_init(&ctx->janitor_lock, NULL);
+ pthread_cond_init(&ctx->janitor_cond, NULL);
+ INIT_LIST_HEAD(&ctx->janitor_fds);
+
+ ret = gf_thread_create(&ctx->janitor, NULL,
+ posix_ctx_janitor_thread_proc, this,
+ "posixctxjan");
if (ret < 0) {
gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED,
@@ -1515,8 +1624,6 @@ posix_spawn_janitor_thread(xlator_t *this)
"thread failed");
goto unlock;
}
-
- priv->janitor_present = _gf_true;
}
}
unlock:
diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c
index dafd1855ef9..d2324823592 100644
--- a/xlators/storage/posix/src/posix-inode-fd-ops.c
+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c
@@ -1236,8 +1236,7 @@ posix_releasedir(xlator_t *this, fd_t *fd)
struct posix_fd *pfd = NULL;
uint64_t tmp_pfd = 0;
int ret = 0;
-
- struct posix_private *priv = NULL;
+ glusterfs_ctx_t *ctx = NULL;
VALIDATE_OR_GOTO(this, out);
VALIDATE_OR_GOTO(fd, out);
@@ -1255,18 +1254,21 @@ posix_releasedir(xlator_t *this, fd_t *fd)
goto out;
}
- priv = this->private;
- if (!priv)
- goto out;
+ ctx = THIS->ctx;
- pthread_mutex_lock(&priv->janitor_lock);
+ pthread_mutex_lock(&ctx->janitor_lock);
{
INIT_LIST_HEAD(&pfd->list);
- list_add_tail(&pfd->list, &priv->janitor_fds);
- pthread_cond_signal(&priv->janitor_cond);
+ list_add_tail(&pfd->list, &ctx->janitor_fds);
+ pthread_cond_signal(&ctx->janitor_cond);
}
- pthread_mutex_unlock(&priv->janitor_lock);
+ pthread_mutex_unlock(&ctx->janitor_lock);
+
+ /*gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir);
+ sys_closedir(pfd->dir);
+ GF_FREE(pfd);
+ */
out:
return 0;
}
@@ -2337,11 +2339,13 @@ posix_release(xlator_t *this, fd_t *fd)
struct posix_fd *pfd = NULL;
int ret = -1;
uint64_t tmp_pfd = 0;
+ glusterfs_ctx_t *ctx = NULL;
VALIDATE_OR_GOTO(this, out);
VALIDATE_OR_GOTO(fd, out);
priv = this->private;
+ ctx = THIS->ctx;
ret = fd_ctx_del(fd, this, &tmp_pfd);
if (ret < 0) {
@@ -2349,22 +2353,23 @@ posix_release(xlator_t *this, fd_t *fd)
"pfd is NULL from fd=%p", fd);
goto out;
}
- pfd = (struct posix_fd *)(long)tmp_pfd;
+ pfd = (struct posix_fd *)(long)tmp_pfd;
if (pfd->dir) {
gf_msg(this->name, GF_LOG_WARNING, 0, P_MSG_DIR_NOT_NULL,
"pfd->dir is %p (not NULL) for file fd=%p", pfd->dir, fd);
}
- if (!priv)
- goto out;
- pthread_mutex_lock(&priv->janitor_lock);
+ pthread_mutex_lock(&ctx->janitor_lock);
{
INIT_LIST_HEAD(&pfd->list);
- list_add_tail(&pfd->list, &priv->janitor_fds);
- pthread_cond_signal(&priv->janitor_cond);
+ list_add_tail(&pfd->list, &ctx->janitor_fds);
+ pthread_cond_signal(&ctx->janitor_cond);
}
- pthread_mutex_unlock(&priv->janitor_lock);
+ pthread_mutex_unlock(&ctx->janitor_lock);
+
+ if (!priv)
+ goto out;
LOCK(&priv->lock);
{
diff --git a/xlators/storage/posix/src/posix-messages.h b/xlators/storage/posix/src/posix-messages.h
index 928723db8f9..32292756bc4 100644
--- a/xlators/storage/posix/src/posix-messages.h
+++ b/xlators/storage/posix/src/posix-messages.h
@@ -68,6 +68,6 @@ GLFS_MSGID(POSIX, P_MSG_XATTR_FAILED, P_MSG_NULL_GFID, P_MSG_FCNTL_FAILED,
P_MSG_FALLOCATE_FAILED, P_MSG_STOREMDATA_FAILED,
P_MSG_FETCHMDATA_FAILED, P_MSG_GETMDATA_FAILED,
P_MSG_SETMDATA_FAILED, P_MSG_FRESHFILE, P_MSG_MUTEX_FAILED,
- P_MSG_COPY_FILE_RANGE_FAILED);
+ P_MSG_COPY_FILE_RANGE_FAILED, P_MSG_TIMER_DELETE_FAILED);
#endif /* !_GLUSTERD_MESSAGES_H_ */
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index a1ec996f4b2..97c9666ab44 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -151,9 +151,6 @@ struct posix_private {
time_t last_landfill_check;
int32_t janitor_sleep_duration;
- struct list_head janitor_fds;
- pthread_cond_t janitor_cond;
- pthread_mutex_t janitor_lock;
int64_t read_value; /* Total read, from init */
int64_t write_value; /* Total write, from init */
@@ -179,9 +176,9 @@ struct posix_private {
*/
gf_boolean_t background_unlink;
- /* janitor thread which cleans up /.trash (created by replicate) */
- pthread_t janitor;
- gf_boolean_t janitor_present;
+ /* janitor task which cleans up /.trash (created by replicate) */
+ struct gf_tw_timer_list *janitor;
+
char *trash_path;
/* lock for brick dir */
DIR *mount_lock;
@@ -344,7 +341,7 @@ int
posix_fhandle_pair(call_frame_t *frame, xlator_t *this, int fd, char *key,
data_t *value, int flags, struct iatt *stbuf, fd_t *_fd);
void
-posix_spawn_janitor_thread(xlator_t *this);
+posix_janitor_timer_start(xlator_t *this);
int
posix_acl_xattr_set(xlator_t *this, const char *path, dict_t *xattr_req);
int
@@ -664,4 +661,7 @@ posix_cs_maintenance(xlator_t *this, fd_t *fd, loc_t *loc, int *pfd,
int
posix_check_dev_file(xlator_t *this, inode_t *inode, char *fop, int *op_errno);
+void
+posix_spawn_ctx_janitor_thread(xlator_t *this);
+
#endif /* _POSIX_H */