diff options
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/Makefile.am | 6 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h | 18 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c | 73 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h | 46 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot-scrub.c | 545 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot-scrub.h | 14 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot-ssm.c | 65 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot-ssm.h | 4 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot.c | 166 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot.h | 70 |
10 files changed, 690 insertions, 317 deletions
diff --git a/xlators/features/bit-rot/src/bitd/Makefile.am b/xlators/features/bit-rot/src/bitd/Makefile.am index 6557943ac69..a915f2d34b8 100644 --- a/xlators/features/bit-rot/src/bitd/Makefile.am +++ b/xlators/features/bit-rot/src/bitd/Makefile.am @@ -9,11 +9,13 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ -I$(CONTRIBDIR)/timer-wheel \ -I$(top_srcdir)/xlators/features/bit-rot/src/stub -bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-tbf.c bit-rot-ssm.c +bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-tbf.c bit-rot-ssm.c \ + bit-rot-scrub-status.c bit_rot_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ $(top_builddir)/xlators/features/changelog/lib/src/libgfchangelog.la -noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-tbf.h bit-rot-bitd-messages.h bit-rot-ssm.h +noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-tbf.h bit-rot-bitd-messages.h bit-rot-ssm.h \ + bit-rot-scrub-status.h AM_CFLAGS = -Wall -DBR_RATE_LIMIT_SIGNER $(GF_CFLAGS) diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h index c0b83c681d7..c6b6a4afa05 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h +++ b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h @@ -40,7 +40,7 @@ */ #define GLFS_BITROT_BITD_BASE GLFS_MSGID_COMP_BITROT_BITD -#define GLFS_BITROT_BITD_NUM_MESSAGES 53 +#define GLFS_BITROT_BITD_NUM_MESSAGES 55 #define GLFS_MSGID_END (GLFS_BITROT_BITD_BASE + \ GLFS_BITROT_BITD_NUM_MESSAGES + 1) /* Messaged with message IDs */ @@ -427,6 +427,22 @@ * */ /*------------*/ +#define BRB_MSG_SSM_FAILED (GLFS_BITROT_BITD_BASE + 54) +/*! + * @messageid + * @diagnosis + * @recommendedaction + * + */ +/*------------*/ +#define BRB_MSG_SCRUB_WAIT_FAILED (GLFS_BITROT_BITD_BASE + 55) +/*! + * @messageid + * @diagnosis + * @recommendedaction + * + */ +/*------------*/ #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages" #endif /* !_BITROT_BITD_MESSAGES_H_ */ diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c new file mode 100644 index 00000000000..0afd7ea05b1 --- /dev/null +++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c @@ -0,0 +1,73 @@ +/* + Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <string.h> + +#include "bit-rot-scrub-status.h" + +void +br_inc_unsigned_file_count (br_scrub_stats_t *scrub_stat) +{ + if (!scrub_stat) + return; + + pthread_mutex_lock (&scrub_stat->lock); + { + scrub_stat->unsigned_files++; + } + pthread_mutex_unlock (&scrub_stat->lock); +} + +void +br_inc_scrubbed_file (br_scrub_stats_t *scrub_stat) +{ + if (!scrub_stat) + return; + + pthread_mutex_lock (&scrub_stat->lock); + { + scrub_stat->scrubbed_files++; + } + pthread_mutex_unlock (&scrub_stat->lock); +} + +void +br_update_scrub_start_time (br_scrub_stats_t *scrub_stat, struct timeval *tv) +{ + if (!scrub_stat) + return; + + pthread_mutex_lock (&scrub_stat->lock); + { + scrub_stat->scrub_start_tv.tv_sec = tv->tv_sec; + } + pthread_mutex_unlock (&scrub_stat->lock); +} + +void +br_update_scrub_finish_time (br_scrub_stats_t *scrub_stat, char *timestr, + struct timeval *tv) +{ + if (!scrub_stat) + return; + + pthread_mutex_lock (&scrub_stat->lock); + { + scrub_stat->scrub_end_tv.tv_sec = tv->tv_sec; + + scrub_stat->scrub_duration = + scrub_stat->scrub_end_tv.tv_sec - + scrub_stat->scrub_start_tv.tv_sec; + + strncpy (scrub_stat->last_scrub_time, timestr, + sizeof (scrub_stat->last_scrub_time)); + } + pthread_mutex_unlock (&scrub_stat->lock); +} diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h new file mode 100644 index 00000000000..694ba0acbe3 --- /dev/null +++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h @@ -0,0 +1,46 @@ +/* + Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __BIT_ROT_SCRUB_STATUS_H__ +#define __BIT_ROT_SCRUB_STATUS_H__ + +#include <stdint.h> +#include <sys/time.h> +#include <pthread.h> + +struct br_scrub_stats { + uint64_t scrubbed_files; /* Total number of scrubbed file */ + + uint64_t unsigned_files; /* Total number of unsigned file */ + + uint64_t scrub_duration; /* Duration of last scrub */ + + char last_scrub_time[1024]; /*last scrub completion time */ + + struct timeval scrub_start_tv; /* Scrubbing starting time*/ + + struct timeval scrub_end_tv; /* Scrubbing finishing time */ + + pthread_mutex_t lock; +}; + +typedef struct br_scrub_stats br_scrub_stats_t; + +void +br_inc_unsigned_file_count (br_scrub_stats_t *scrub_stat); +void +br_inc_scrubbed_file (br_scrub_stats_t *scrub_stat); +void +br_update_scrub_start_time (br_scrub_stats_t *scrub_stat, struct timeval *tv); +void +br_update_scrub_finish_time (br_scrub_stats_t *scrub_stat, char *timestr, + struct timeval *tv); + +#endif /* __BIT_ROT_SCRUB_STATUS_H__ */ diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c index d95008ca9a2..c0421d0ac02 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c +++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c @@ -19,6 +19,7 @@ #include "bit-rot-scrub.h" #include <pthread.h> #include "bit-rot-bitd-messages.h" +#include "bit-rot-scrub-status.h" struct br_scrubbers { pthread_t scrubthread; @@ -74,20 +75,6 @@ bitd_fetch_signature (xlator_t *this, br_child_t *child, } -static void -br_inc_unsigned_file_count (xlator_t *this) -{ - br_private_t *priv = NULL; - - priv = this->private; - - pthread_mutex_lock (&priv->scrub_stat.lock); - { - priv->scrub_stat.unsigned_files++; - } - pthread_mutex_unlock (&priv->scrub_stat.lock); -} - /** * POST COMPUTE CHECK * @@ -101,7 +88,8 @@ int32_t bitd_scrub_post_compute_check (xlator_t *this, br_child_t *child, fd_t *fd, unsigned long version, - br_isignature_out_t **signature) + br_isignature_out_t **signature, + br_scrub_stats_t *scrub_stat) { int32_t ret = 0; size_t signlen = 0; @@ -109,8 +97,10 @@ bitd_scrub_post_compute_check (xlator_t *this, br_isignature_out_t *signptr = NULL; ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr); - if (ret < 0) + if (ret < 0) { + br_inc_unsigned_file_count (scrub_stat); goto out; + } /** * Either the object got dirtied during the time the signature was @@ -121,7 +111,7 @@ bitd_scrub_post_compute_check (xlator_t *this, * The log entry looks pretty ugly, but helps in debugging.. */ if (signptr->stale || (signptr->version != version)) { - br_inc_unsigned_file_count (this); + br_inc_unsigned_file_count (scrub_stat); gf_msg_debug (this->name, 0, "<STAGE: POST> Object [GFID: %s] " "either has a stale signature OR underwent " "signing during checksumming {Stale: %d | " @@ -149,15 +139,18 @@ bitd_scrub_post_compute_check (xlator_t *this, static int32_t bitd_signature_staleness (xlator_t *this, br_child_t *child, fd_t *fd, - int *stale, unsigned long *version) + int *stale, unsigned long *version, + br_scrub_stats_t *scrub_stat) { int32_t ret = -1; dict_t *xattr = NULL; br_isignature_out_t *signptr = NULL; ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr); - if (ret < 0) + if (ret < 0) { + br_inc_unsigned_file_count (scrub_stat); goto out; + } /** * save verison for validation in post compute stage @@ -182,7 +175,8 @@ bitd_signature_staleness (xlator_t *this, */ int32_t bitd_scrub_pre_compute_check (xlator_t *this, br_child_t *child, - fd_t *fd, unsigned long *version) + fd_t *fd, unsigned long *version, + br_scrub_stats_t *scrub_stat) { int stale = 0; int32_t ret = -1; @@ -194,9 +188,10 @@ bitd_scrub_pre_compute_check (xlator_t *this, br_child_t *child, goto out; } - ret = bitd_signature_staleness (this, child, fd, &stale, version); + ret = bitd_signature_staleness (this, child, fd, &stale, version, + scrub_stat); if (!ret && stale) { - br_inc_unsigned_file_count (this); + br_inc_unsigned_file_count (scrub_stat); gf_msg_debug (this->name, 0, "<STAGE: PRE> Object [GFID: %s] " "has stale signature", uuid_utoa (fd->inode->gfid)); @@ -272,16 +267,6 @@ bitd_compare_ckum (xlator_t *this, return ret; } -static void -br_inc_scrubbed_file (br_private_t *priv) -{ - pthread_mutex_lock (&priv->scrub_stat.lock); - { - priv->scrub_stat.scrubbed_files++; - } - pthread_mutex_unlock (&priv->scrub_stat.lock); -} - /** * "The Scrubber" * @@ -374,7 +359,8 @@ br_scrubber_scrub_begin (xlator_t *this, struct br_fsscan_entry *fsentry) * - presence of bad object * - signature staleness */ - ret = bitd_scrub_pre_compute_check (this, child, fd, &signedversion); + ret = bitd_scrub_pre_compute_check (this, child, fd, &signedversion, + &priv->scrub_stat); if (ret) goto unrefd; /* skip this object */ @@ -397,8 +383,8 @@ br_scrubber_scrub_begin (xlator_t *this, struct br_fsscan_entry *fsentry) * perform post compute checks as an object's signature may have * become stale while scrubber calculated checksum. */ - ret = bitd_scrub_post_compute_check (this, child, - fd, signedversion, &sign); + ret = bitd_scrub_post_compute_check (this, child, fd, signedversion, + &sign, &priv->scrub_stat); if (ret) goto free_md; @@ -406,7 +392,7 @@ br_scrubber_scrub_begin (xlator_t *this, struct br_fsscan_entry *fsentry) linked_inode, entry, fd, child, &loc); /* Increment of total number of scrubbed file counter */ - br_inc_scrubbed_file (priv); + br_inc_scrubbed_file (&priv->scrub_stat); GF_FREE (sign); /* alloced on post-compute */ @@ -560,171 +546,215 @@ br_fsscanner_handle_entry (xlator_t *subvol, } int32_t -br_fsscan_deactivate (xlator_t *this, br_child_t *child) +br_fsscan_deactivate (xlator_t *this) { int ret = 0; br_private_t *priv = NULL; br_scrub_state_t nstate = 0; - struct br_scanfs *fsscan = NULL; + struct br_monitor *scrub_monitor = NULL; priv = this->private; - fsscan = &child->fsscan; + scrub_monitor = &priv->scrub_monitor; - ret = gf_tw_del_timer (priv->timer_wheel, fsscan->timer); + ret = gf_tw_del_timer (priv->timer_wheel, scrub_monitor->timer); if (ret == 0) { nstate = BR_SCRUB_STATE_STALLED; gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, - "Brick [%s] is under active scrubbing. Pausing scrub..", - child->brick_path); + "Volume is under active scrubbing. Pausing scrub.."); } else { nstate = BR_SCRUB_STATE_PAUSED; gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, - "Scrubber paused [Brick: %s]", child->brick_path); + "Scrubber paused"); } - _br_child_set_scrub_state (child, nstate); + _br_monitor_set_scrub_state (scrub_monitor, nstate); return 0; } + static void -br_update_scrub_start_time (xlator_t *this, struct timeval *tv) +br_scrubber_log_time (xlator_t *this, const char *sfx) { - br_private_t *priv = NULL; - static int child; + char timestr[1024] = {0,}; + struct timeval tv = {0,}; + br_private_t *priv = NULL; priv = this->private; + gettimeofday (&tv, NULL); + gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT); - /* Setting scrubber starting time for first child only */ - if (child == 0) { - pthread_mutex_lock (&priv->scrub_stat.lock); - { - priv->scrub_stat.scrub_start_tv.tv_sec = tv->tv_sec; - } - pthread_mutex_unlock (&priv->scrub_stat.lock); + if (strcasecmp (sfx, "started") == 0) { + br_update_scrub_start_time (&priv->scrub_stat, &tv); + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START, + "Scrubbing %s at %s", sfx, timestr); + } else { + br_update_scrub_finish_time (&priv->scrub_stat, timestr, &tv); + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH, + "Scrubbing %s at %s", sfx, timestr); } +} - if (++child == priv->up_children) { - child = 0; +static void +br_fsscanner_log_time (xlator_t *this, br_child_t *child, const char *sfx) +{ + char timestr[1024] = {0,}; + struct timeval tv = {0,}; + + gettimeofday (&tv, NULL); + gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT); + + if (strcasecmp (sfx, "started") == 0) { + gf_msg_debug (this->name, 0, "Scrubbing \"%s\" %s at %s", + child->brick_path, sfx, timestr); + } else { + gf_msg_debug (this->name, 0, "Scrubbing \"%s\" %s at %s", + child->brick_path, sfx, timestr); } } +void +br_child_set_scrub_state (br_child_t *child, gf_boolean_t state) +{ + child->active_scrubbing = state; +} + static void -br_update_scrub_finish_time (xlator_t *this, char *timestr, struct timeval *tv) +br_fsscanner_wait_until_kicked (xlator_t *this, br_child_t *child) { - br_private_t *priv = NULL; - static int child; + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; priv = this->private; + scrub_monitor = &priv->scrub_monitor; - /*Setting scrubber finishing time at time time of last child operation*/ - if (++child == priv->up_children) { - pthread_mutex_lock (&priv->scrub_stat.lock); - { - priv->scrub_stat.scrub_end_tv.tv_sec = tv->tv_sec; - - priv->scrub_stat.scrub_duration = - priv->scrub_stat.scrub_end_tv.tv_sec - - priv->scrub_stat.scrub_start_tv.tv_sec; - - strncpy (priv->scrub_stat.last_scrub_time, timestr, - sizeof (priv->scrub_stat.last_scrub_time)); + pthread_cleanup_push (_br_lock_cleaner, &scrub_monitor->wakelock); + pthread_mutex_lock (&scrub_monitor->wakelock); + { + while (!scrub_monitor->kick) + pthread_cond_wait (&scrub_monitor->wakecond, + &scrub_monitor->wakelock); - child = 0; + /* Child lock is to synchronize with disconnect events */ + pthread_cleanup_push (_br_lock_cleaner, &child->lock); + LOCK (&child->lock); + { + scrub_monitor->active_child_count++; + br_child_set_scrub_state (child, _gf_true); } - pthread_mutex_unlock (&priv->scrub_stat.lock); + UNLOCK (&child->lock); + pthread_cleanup_pop (0); } + pthread_mutex_unlock (&scrub_monitor->wakelock); + pthread_cleanup_pop (0); } static void -br_fsscanner_log_time (xlator_t *this, br_child_t *child, const char *sfx) +br_scrubber_entry_control (xlator_t *this) { - char timestr[1024] = {0,}; - struct timeval tv = {0,}; + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; - gettimeofday (&tv, NULL); - gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT); + priv = this->private; + scrub_monitor = &priv->scrub_monitor; - if (strcasecmp (sfx, "started") == 0) { - br_update_scrub_start_time (this, &tv); - gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START, - "Scrubbing \"%s\" %s at %s", child->brick_path, sfx, - timestr); - } else { - br_update_scrub_finish_time (this, timestr, &tv); - gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH, - "Scrubbing \"%s\" %s at %s", child->brick_path, sfx, - timestr); + LOCK (&scrub_monitor->lock); + { + /* Move the state to BR_SCRUB_STATE_ACTIVE */ + if (scrub_monitor->state == BR_SCRUB_STATE_PENDING) + scrub_monitor->state = BR_SCRUB_STATE_ACTIVE; + br_scrubber_log_time (this, "started"); } + UNLOCK (&scrub_monitor->lock); } static void -br_fsscanner_wait_until_kicked (xlator_t *this, struct br_scanfs *fsscan) +br_scrubber_exit_control (xlator_t *this) { - static int i; - br_private_t *priv = NULL; + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; priv = this->private; + scrub_monitor = &priv->scrub_monitor; - pthread_cleanup_push (_br_lock_cleaner, &fsscan->wakelock); - pthread_mutex_lock (&fsscan->wakelock); + LOCK (&scrub_monitor->lock); { - while (!fsscan->kick) - pthread_cond_wait (&fsscan->wakecond, - &fsscan->wakelock); - - /* resetting total number of scrubbed file when scrubbing - * done for all of its children */ - if (i == priv->up_children) { - pthread_mutex_lock (&priv->scrub_stat.lock); - { - priv->scrub_stat.scrubbed_files = 0; - priv->scrub_stat.unsigned_files = 0; - i = 0; - } - pthread_mutex_unlock (&priv->scrub_stat.lock); - } - ++i; + br_scrubber_log_time (this, "finished"); - fsscan->kick = _gf_false; + if (scrub_monitor->state == BR_SCRUB_STATE_ACTIVE) { + (void) br_fsscan_activate (this); + } else { + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, + "Volume waiting to get rescheduled.."); + } } - pthread_mutex_unlock (&fsscan->wakelock); - pthread_cleanup_pop (0); + UNLOCK (&scrub_monitor->lock); } static void br_fsscanner_entry_control (xlator_t *this, br_child_t *child) { - struct br_scanfs *fsscan = &child->fsscan; - - LOCK (&child->lock); - { - if (fsscan->state == BR_SCRUB_STATE_PENDING) - fsscan->state = BR_SCRUB_STATE_ACTIVE; br_fsscanner_log_time (this, child, "started"); - } - UNLOCK (&child->lock); } static void br_fsscanner_exit_control (xlator_t *this, br_child_t *child) { - struct br_scanfs *fsscan = &child->fsscan; + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; - LOCK (&child->lock); + priv = this->private; + scrub_monitor = &priv->scrub_monitor; + + if (!_br_is_child_connected (child)) { + gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SCRUB_INFO, + "Brick [%s] disconnected while scrubbing. Scrubbing " + "might be incomplete", child->brick_path); + } + + br_fsscanner_log_time (this, child, "finished"); + + pthread_cleanup_push (_br_lock_cleaner, &scrub_monitor->wakelock); + pthread_mutex_lock (&scrub_monitor->wakelock); { - fsscan->over = _gf_true; - br_fsscanner_log_time (this, child, "finished"); + scrub_monitor->active_child_count--; + pthread_cleanup_push (_br_lock_cleaner, &child->lock); + LOCK (&child->lock); + { + br_child_set_scrub_state (child, _gf_false); + } + UNLOCK (&child->lock); + pthread_cleanup_pop (0); - if (fsscan->state == BR_SCRUB_STATE_ACTIVE) { - (void) br_fsscan_activate (this, child); + if (scrub_monitor->active_child_count == 0) { + /* The last child has finished scrubbing. + * Set the kick to false and wake up other + * children who are waiting for the last + * child to complete scrubbing. + */ + scrub_monitor->kick = _gf_false; + pthread_cond_broadcast (&scrub_monitor->wakecond); + + /* Signal monitor thread waiting for the all + * the children to finish scrubbing. + */ + pthread_cleanup_push (_br_lock_cleaner, + &scrub_monitor->donelock); + pthread_mutex_lock (&scrub_monitor->donelock); + { + scrub_monitor->done = _gf_true; + pthread_cond_signal (&scrub_monitor->donecond); + } + pthread_mutex_unlock (&scrub_monitor->donelock); + pthread_cleanup_pop (0); } else { - gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, - "Brick [%s] waiting to get rescheduled..", - child->brick_path); + while (scrub_monitor->active_child_count) + pthread_cond_wait (&scrub_monitor->wakecond, + &scrub_monitor->wakelock); } } - UNLOCK (&child->lock); + pthread_mutex_unlock (&scrub_monitor->wakelock); + pthread_cleanup_pop (0); } void * @@ -743,7 +773,7 @@ br_fsscanner (void *arg) loc.inode = child->table->root; while (1) { - br_fsscanner_wait_until_kicked (this, fsscan); + br_fsscanner_wait_until_kicked (this, child); { /* precursor for scrub */ br_fsscanner_entry_control (this, child); @@ -775,22 +805,29 @@ br_kickstart_scanner (struct gf_tw_timer_list *timer, void *data, unsigned long calltime) { xlator_t *this = NULL; - br_child_t *child = data; - struct br_scanfs *fsscan = NULL; + struct br_monitor *scrub_monitor = data; + br_private_t *priv = NULL; - THIS = this = child->this; - fsscan = &child->fsscan; + THIS = this = scrub_monitor->this; + priv = this->private; + + /* Reset scrub statistics */ + priv->scrub_stat.scrubbed_files = 0; + priv->scrub_stat.unsigned_files = 0; + + /* Moves state from PENDING to ACTIVE */ + (void) br_scrubber_entry_control (this); /* kickstart scanning.. */ - pthread_mutex_lock (&fsscan->wakelock); + pthread_mutex_lock (&scrub_monitor->wakelock); { - fsscan->kick = _gf_true; - pthread_cond_signal (&fsscan->wakecond); + scrub_monitor->kick = _gf_true; + GF_ASSERT (scrub_monitor->active_child_count == 0); + pthread_cond_broadcast (&scrub_monitor->wakecond); } - pthread_mutex_unlock (&fsscan->wakelock); + pthread_mutex_unlock (&scrub_monitor->wakelock); return; - } static uint32_t @@ -834,22 +871,22 @@ br_fsscan_calculate_timeout (scrub_freq_t freq) } int32_t -br_fsscan_schedule (xlator_t *this, br_child_t *child) +br_fsscan_schedule (xlator_t *this) { uint32_t timo = 0; br_private_t *priv = NULL; struct timeval tv = {0,}; char timestr[1024] = {0,}; - struct br_scanfs *fsscan = NULL; struct br_scrubber *fsscrub = NULL; struct gf_tw_timer_list *timer = NULL; + struct br_monitor *scrub_monitor = NULL; priv = this->private; - fsscan = &child->fsscan; fsscrub = &priv->fsscrub; + scrub_monitor = &priv->scrub_monitor; (void) gettimeofday (&tv, NULL); - fsscan->boot = tv.tv_sec; + scrub_monitor->boot = tv.tv_sec; timo = br_fsscan_calculate_timeout (fsscrub->frequency); if (timo == 0) { @@ -858,25 +895,25 @@ br_fsscan_schedule (xlator_t *this, br_child_t *child) goto error_return; } - fsscan->timer = GF_CALLOC (1, sizeof (*fsscan->timer), + scrub_monitor->timer = GF_CALLOC (1, sizeof (*scrub_monitor->timer), gf_br_stub_mt_br_scanner_freq_t); - if (!fsscan->timer) + if (!scrub_monitor->timer) goto error_return; - timer = fsscan->timer; + timer = scrub_monitor->timer; INIT_LIST_HEAD (&timer->entry); - timer->data = child; + timer->data = scrub_monitor; timer->expires = timo; timer->function = br_kickstart_scanner; gf_tw_add_timer (priv->timer_wheel, timer); - _br_child_set_scrub_state (child, BR_SCRUB_STATE_PENDING); + _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING); gf_time_fmt (timestr, sizeof (timestr), - (fsscan->boot + timo), gf_timefmt_FT); - gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing for " - "%s scheduled to run at %s", child->brick_path, timestr); + (scrub_monitor->boot + timo), gf_timefmt_FT); + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing is " + "scheduled to run at %s", timestr); return 0; @@ -885,18 +922,18 @@ br_fsscan_schedule (xlator_t *this, br_child_t *child) } int32_t -br_fsscan_activate (xlator_t *this, br_child_t *child) +br_fsscan_activate (xlator_t *this) { uint32_t timo = 0; char timestr[1024] = {0,}; struct timeval now = {0,}; br_private_t *priv = NULL; - struct br_scanfs *fsscan = NULL; struct br_scrubber *fsscrub = NULL; + struct br_monitor *scrub_monitor = NULL; priv = this->private; - fsscan = &child->fsscan; fsscrub = &priv->fsscrub; + scrub_monitor = &priv->scrub_monitor; (void) gettimeofday (&now, NULL); timo = br_fsscan_calculate_timeout (fsscrub->frequency); @@ -906,32 +943,37 @@ br_fsscan_activate (xlator_t *this, br_child_t *child) return -1; } - fsscan->over = _gf_false; + pthread_mutex_lock (&scrub_monitor->donelock); + { + scrub_monitor->done = _gf_false; + } + pthread_mutex_unlock (&scrub_monitor->donelock); + gf_time_fmt (timestr, sizeof (timestr), (now.tv_sec + timo), gf_timefmt_FT); - (void) gf_tw_mod_timer (priv->timer_wheel, fsscan->timer, timo); + (void) gf_tw_mod_timer (priv->timer_wheel, scrub_monitor->timer, timo); - _br_child_set_scrub_state (child, BR_SCRUB_STATE_PENDING); - gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing for " - "%s rescheduled to run at %s", child->brick_path, timestr); + _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING); + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing is " + "rescheduled to run at %s", timestr); return 0; } int32_t -br_fsscan_reschedule (xlator_t *this, br_child_t *child) +br_fsscan_reschedule (xlator_t *this) { int32_t ret = 0; uint32_t timo = 0; char timestr[1024] = {0,}; struct timeval now = {0,}; br_private_t *priv = NULL; - struct br_scanfs *fsscan = NULL; struct br_scrubber *fsscrub = NULL; + struct br_monitor *scrub_monitor = NULL; priv = this->private; - fsscan = &child->fsscan; fsscrub = &priv->fsscrub; + scrub_monitor = &priv->scrub_monitor; if (!fsscrub->frequency_reconf) return 0; @@ -947,17 +989,21 @@ br_fsscan_reschedule (xlator_t *this, br_child_t *child) gf_time_fmt (timestr, sizeof (timestr), (now.tv_sec + timo), gf_timefmt_FT); - fsscan->over = _gf_false; - ret = gf_tw_mod_timer_pending (priv->timer_wheel, fsscan->timer, timo); + pthread_mutex_lock (&scrub_monitor->donelock); + { + scrub_monitor->done = _gf_false; + } + pthread_mutex_unlock (&scrub_monitor->donelock); + + ret = gf_tw_mod_timer_pending (priv->timer_wheel, scrub_monitor->timer, timo); if (ret == 0) gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, - "Scrubber for %s is currently running and would be " - "rescheduled after completion", child->brick_path); + "Scrubber is currently running and would be " + "rescheduled after completion"); else { - _br_child_set_scrub_state (child, BR_SCRUB_STATE_PENDING); + _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING); gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, - "Scrubbing for %s rescheduled to run at %s", - child->brick_path, timestr); + "Scrubbing rescheduled to run at %s", timestr); } return 0; @@ -1723,15 +1769,174 @@ out: return ret; } +static int +wait_for_scrub_to_finish (xlator_t *this) +{ + int ret = -1; + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; + + priv = this->private; + scrub_monitor = &priv->scrub_monitor; + + GF_VALIDATE_OR_GOTO ("bit-rot", scrub_monitor, out); + GF_VALIDATE_OR_GOTO ("bit-rot", this, out); + + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, + "Waiting for all children to start and finish scrub"); + + pthread_mutex_lock (&scrub_monitor->donelock); + { + while (!scrub_monitor->done) + pthread_cond_wait (&scrub_monitor->donecond, + &scrub_monitor->donelock); + } + pthread_mutex_unlock (&scrub_monitor->donelock); + ret = 0; +out: + return ret; +} + +/** + * This function is executed in a separate thread. This is scrubber monitor + * thread that takes care of state machine. + */ +void * +br_monitor_thread (void *arg) +{ + int32_t ret = 0; + xlator_t *this = NULL; + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; + + this = arg; + priv = this->private; + + /* + * Since, this is the topmost xlator, THIS has to be set by bit-rot + * xlator itself (STACK_WIND wont help in this case). Also it has + * to be done for each thread that gets spawned. Otherwise, a new + * thread will get global_xlator's pointer when it does "THIS". + */ + THIS = this; + + scrub_monitor = &priv->scrub_monitor; + + pthread_mutex_lock (&scrub_monitor->mutex); + { + while (!scrub_monitor->inited) + pthread_cond_wait (&scrub_monitor->cond, + &scrub_monitor->mutex); + } + pthread_mutex_unlock (&scrub_monitor->mutex); + + /* this needs to be serialized with reconfigure() */ + pthread_mutex_lock (&priv->lock); + { + ret = br_scrub_state_machine (this); + } + pthread_mutex_unlock (&priv->lock); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, -ret, + BRB_MSG_SSM_FAILED, + "Scrub state machine failed"); + goto out; + } + + while (1) { + /* Wait for all children to finish scrubbing */ + ret = wait_for_scrub_to_finish (this); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, -ret, + BRB_MSG_SCRUB_WAIT_FAILED, + "Scrub wait failed"); + goto out; + } + + /* scrub exit criteria: Move the state to PENDING */ + br_scrubber_exit_control (this); + } + +out: + return NULL; +} + +static void +br_set_scrub_state (struct br_monitor *scrub_monitor, br_scrub_state_t state) +{ + LOCK (&scrub_monitor->lock); + { + _br_monitor_set_scrub_state (scrub_monitor, state); + } + UNLOCK (&scrub_monitor->lock); +} + +int32_t +br_scrubber_monitor_init (xlator_t *this, br_private_t *priv) +{ + struct br_monitor *scrub_monitor = NULL; + int ret = 0; + + scrub_monitor = &priv->scrub_monitor; + + LOCK_INIT (&scrub_monitor->lock); + scrub_monitor->this = this; + + scrub_monitor->inited = _gf_false; + pthread_mutex_init (&scrub_monitor->mutex, NULL); + pthread_cond_init (&scrub_monitor->cond, NULL); + + scrub_monitor->kick = _gf_false; + scrub_monitor->active_child_count = 0; + pthread_mutex_init (&scrub_monitor->wakelock, NULL); + pthread_cond_init (&scrub_monitor->wakecond, NULL); + + scrub_monitor->done = _gf_false; + pthread_mutex_init (&scrub_monitor->donelock, NULL); + pthread_cond_init (&scrub_monitor->donecond, NULL); + + /* Set the state to INACTIVE */ + br_set_scrub_state (&priv->scrub_monitor, BR_SCRUB_STATE_INACTIVE); + + /* Start the monitor thread */ + ret = gf_thread_create (&scrub_monitor->thread, NULL, br_monitor_thread, this); + if (ret != 0) { + gf_msg (this->name, GF_LOG_ERROR, -ret, + BRB_MSG_SPAWN_FAILED, "monitor thread creation failed"); + ret = -1; + goto err; + } + + return 0; +err: + pthread_mutex_destroy (&scrub_monitor->mutex); + pthread_cond_destroy (&scrub_monitor->cond); + + pthread_mutex_destroy (&scrub_monitor->wakelock); + pthread_cond_destroy (&scrub_monitor->wakecond); + + pthread_mutex_destroy (&scrub_monitor->donelock); + pthread_cond_destroy (&scrub_monitor->donecond); + + LOCK_DESTROY (&scrub_monitor->lock); + + return ret; +} + int32_t br_scrubber_init (xlator_t *this, br_private_t *priv) { struct br_scrubber *fsscrub = NULL; + int ret = 0; priv->tbf = br_tbf_init (NULL, 0); if (!priv->tbf) return -1; + ret = br_scrubber_monitor_init (this, priv); + if (ret) + return -1; + fsscrub = &priv->fsscrub; fsscrub->this = this; diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h index b27decd41c1..63169068ed4 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h +++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h @@ -16,15 +16,21 @@ void *br_fsscanner (void *); -int32_t br_fsscan_schedule (xlator_t *, br_child_t *); -int32_t br_fsscan_reschedule (xlator_t *, br_child_t *); -int32_t br_fsscan_activate (xlator_t *, br_child_t *); -int32_t br_fsscan_deactivate (xlator_t *, br_child_t *); +int32_t br_fsscan_schedule (xlator_t *); +int32_t br_fsscan_reschedule (xlator_t *); +int32_t br_fsscan_activate (xlator_t *); +int32_t br_fsscan_deactivate (xlator_t *); int32_t br_scrubber_handle_options (xlator_t *, br_private_t *, dict_t *); +int32_t +br_scrubber_monitor_init (xlator_t *, br_private_t *); + int32_t br_scrubber_init (xlator_t *, br_private_t *); int32_t br_collect_bad_objects_from_children (xlator_t *this, dict_t *dict); +void +br_child_set_scrub_state (br_child_t *, gf_boolean_t); + #endif /* __BIT_ROT_SCRUB_H__ */ diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c index fcffc04feda..d304fc804ee 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c +++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c @@ -12,52 +12,73 @@ #include "bit-rot-scrub.h" #include "bit-rot-bitd-messages.h" -int br_scrub_ssm_noop (xlator_t *this, br_child_t *child) +int br_scrub_ssm_noop (xlator_t *this) { return 0; } int -br_scrub_ssm_state_pause (xlator_t *this, br_child_t *child) +br_scrub_ssm_state_pause (xlator_t *this) { + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; + + priv = this->private; + scrub_monitor = &priv->scrub_monitor; + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO, - "Scrubber paused [Brick: %s]", child->brick_path); - _br_child_set_scrub_state (child, BR_SCRUB_STATE_PAUSED); + "Scrubber paused"); + _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PAUSED); return 0; } int -br_scrub_ssm_state_ipause (xlator_t *this, br_child_t *child) +br_scrub_ssm_state_ipause (xlator_t *this) { + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; + + priv = this->private; + scrub_monitor = &priv->scrub_monitor; + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO, - "Scrubber paused [Brick: %s]", child->brick_path); - _br_child_set_scrub_state (child, BR_SCRUB_STATE_IPAUSED); + "Scrubber paused"); + _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_IPAUSED); return 0; } int -br_scrub_ssm_state_active (xlator_t *this, br_child_t *child) +br_scrub_ssm_state_active (xlator_t *this) { - struct br_scanfs *fsscan = &child->fsscan; + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; - if (fsscan->over) { - (void) br_fsscan_activate (this, child); + priv = this->private; + scrub_monitor = &priv->scrub_monitor; + + if (scrub_monitor->done) { + (void) br_fsscan_activate (this); } else { gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO, - "Scrubbing resumed [Brick %s]", child->brick_path); - _br_child_set_scrub_state (child, BR_SCRUB_STATE_ACTIVE); + "Scrubbing resumed"); + _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_ACTIVE); } return 0; } int -br_scrub_ssm_state_stall (xlator_t *this, br_child_t *child) +br_scrub_ssm_state_stall (xlator_t *this) { + br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; + + priv = this->private; + scrub_monitor = &priv->scrub_monitor; + gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO, - "Brick [%s] is under active scrubbing. Pausing scrub..", - child->brick_path); - _br_child_set_scrub_state (child, BR_SCRUB_STATE_STALLED); + "Volume is under active scrubbing. Pausing scrub.."); + _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_STALLED); return 0; } @@ -72,22 +93,22 @@ br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] = { }; int32_t -br_scrub_state_machine (xlator_t *this, br_child_t *child) +br_scrub_state_machine (xlator_t *this) { br_private_t *priv = NULL; br_scrub_ssm_call *call = NULL; - struct br_scanfs *fsscan = NULL; struct br_scrubber *fsscrub = NULL; br_scrub_state_t currstate = 0; br_scrub_event_t event = 0; + struct br_monitor *scrub_monitor = NULL; priv = this->private; - fsscan = &child->fsscan; fsscrub = &priv->fsscrub; + scrub_monitor = &priv->scrub_monitor; - currstate = fsscan->state; + currstate = scrub_monitor->state; event = _br_child_get_scrub_event (fsscrub); call = br_scrub_ssm[currstate][event]; - return call (this, child); + return call (this); } diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h index 72fd62b3630..936ee4d837c 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h +++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h @@ -29,8 +29,8 @@ typedef enum br_scrub_event { BR_SCRUB_MAXEVENTS, } br_scrub_event_t; -struct br_child; +struct br_monitor; -int32_t br_scrub_state_machine (xlator_t *, struct br_child *); +int32_t br_scrub_state_machine (xlator_t *); #endif /* __BIT_ROT_SSM_H__ */ diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c index e4d9113bd25..7d337bf724f 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot.c +++ b/xlators/features/bit-rot/src/bitd/bit-rot.c @@ -1099,16 +1099,6 @@ br_set_child_state (br_child_t *child, br_child_state_t state) UNLOCK (&child->lock); } -static void -br_set_scrub_state (br_child_t *child, br_scrub_state_t state) -{ - LOCK (&child->lock); - { - _br_child_set_scrub_state (child, state); - } - UNLOCK (&child->lock); -} - /** * At this point a thread is spawned to crawl the filesystem (in * tortoise pace) to sign objects that were not signed in previous run(s). @@ -1168,11 +1158,11 @@ br_launch_scrubber (xlator_t *this, br_child_t *child, { int32_t ret = -1; br_private_t *priv = NULL; + struct br_monitor *scrub_monitor = NULL; priv = this->private; - fsscan->kick = _gf_false; - fsscan->over = _gf_false; + scrub_monitor = &priv->scrub_monitor; ret = gf_thread_create (&child->thread, NULL, br_fsscanner, child); if (ret != 0) { gf_msg (this->name, GF_LOG_ALERT, 0, BRB_MSG_SPAWN_FAILED, @@ -1181,14 +1171,14 @@ br_launch_scrubber (xlator_t *this, br_child_t *child, goto error_return; } - /* this needs to be serialized with reconfigure() */ - pthread_mutex_lock (&priv->lock); + /* Signal monitor to kick off state machine*/ + pthread_mutex_lock (&scrub_monitor->mutex); { - ret = br_scrub_state_machine (this, child); + if (!scrub_monitor->inited) + pthread_cond_signal (&scrub_monitor->cond); + scrub_monitor->inited = _gf_true; } - pthread_mutex_unlock (&priv->lock); - if (ret) - goto cleanup_thread; + pthread_mutex_unlock (&scrub_monitor->mutex); /** * Everything has been setup.. add this subvolume to scrubbers @@ -1203,8 +1193,6 @@ br_launch_scrubber (xlator_t *this, br_child_t *child, return 0; - cleanup_thread: - (void) gf_thread_cleanup_xint (child->thread); error_return: return -1; } @@ -1237,10 +1225,6 @@ br_enact_scrubber (xlator_t *this, br_child_t *child) INIT_LIST_HEAD (&fsscan->queued); INIT_LIST_HEAD (&fsscan->ready); - /* init scheduler related variables */ - pthread_mutex_init (&fsscan->wakelock, NULL); - pthread_cond_init (&fsscan->wakecond, NULL); - ret = br_launch_scrubber (this, child, fsscan, fsscrub); if (ret) goto error_return; @@ -1303,6 +1287,7 @@ br_brick_connect (xlator_t *this, br_child_t *child) GF_VALIDATE_OR_GOTO (this->name, child, out); GF_VALIDATE_OR_GOTO (this->name, this->private, out); + br_child_set_scrub_state (child, _gf_false); br_set_child_state (child, BR_CHILD_STATE_INITIALIZING); loc.inode = inode_ref (child->table->root); @@ -1364,12 +1349,17 @@ br_cleanup_scrubber (xlator_t *this, br_child_t *child) { int32_t ret = 0; br_private_t *priv = NULL; - struct br_scanfs *fsscan = NULL; struct br_scrubber *fsscrub = NULL; + struct br_monitor *scrub_monitor = NULL; priv = this->private; - fsscan = &child->fsscan; fsscrub = &priv->fsscrub; + scrub_monitor = &priv->scrub_monitor; + + if (_br_is_child_scrub_active (child)) { + scrub_monitor->active_child_count--; + br_child_set_scrub_state (child, _gf_false); + } /** * 0x0: child (brick) goes out of rotation @@ -1401,21 +1391,6 @@ br_cleanup_scrubber (xlator_t *this, br_child_t *child) 0, BRB_MSG_SCRUB_THREAD_CLEANUP, "Error cleaning up scanner thread"); - /** - * 0x2: free()up resources - */ - if (fsscan->timer) { - (void) gf_tw_del_timer (priv->timer_wheel, fsscan->timer); - - GF_FREE (fsscan->timer); - fsscan->timer = NULL; - } - - /** - * 0x3: reset scrubber state - */ - _br_child_set_scrub_state (child, BR_SCRUB_STATE_INACTIVE); - gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUBBER_CLEANED, "Cleaned up scrubber for brick [%s]", child->brick_path); @@ -1432,23 +1407,33 @@ int32_t br_brick_disconnect (xlator_t *this, br_child_t *child) { int32_t ret = 0; + struct br_monitor *scrub_monitor = NULL; br_private_t *priv = this->private; - LOCK (&child->lock); + scrub_monitor = &priv->scrub_monitor; + + /* Lock order should be wakelock and then child lock to + * dead locks. + */ + pthread_mutex_lock (&scrub_monitor->wakelock); { - if (!_br_is_child_connected (child)) - goto unblock; + LOCK (&child->lock); + { + if (!_br_is_child_connected (child)) + goto unblock; - /* child is on death row.. */ - _br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED); + /* child is on death row.. */ + _br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED); - if (priv->iamscrubber) - ret = br_cleanup_scrubber (this, child); - else - ret = br_cleanup_signer (this, child); - } + if (priv->iamscrubber) + ret = br_cleanup_scrubber (this, child); + else + ret = br_cleanup_signer (this, child); + } unblock: - UNLOCK (&child->lock); + UNLOCK (&child->lock); + } + pthread_mutex_unlock (&scrub_monitor->wakelock); return ret; } @@ -1569,7 +1554,7 @@ br_scrubber_status_get (xlator_t *this, dict_t **dict) memset (key, 0, 256); snprintf (key, 256, "scrubbed-files"); - ret = dict_set_uint32 (*dict, key, scrub_stats->scrubbed_files); + ret = dict_set_uint64 (*dict, key, scrub_stats->scrubbed_files); if (ret) { gf_msg_debug (this->name, 0, "Failed to setting scrubbed file " "entry to the dictionary"); @@ -1577,7 +1562,7 @@ br_scrubber_status_get (xlator_t *this, dict_t **dict) memset (key, 0, 256); snprintf (key, 256, "unsigned-files"); - ret = dict_set_uint32 (*dict, key, scrub_stats->unsigned_files); + ret = dict_set_uint64 (*dict, key, scrub_stats->unsigned_files); if (ret) { gf_msg_debug (this->name, 0, "Failed to set unsigned file count" " entry to the dictionary"); @@ -1585,7 +1570,7 @@ br_scrubber_status_get (xlator_t *this, dict_t **dict) memset (key, 0, 256); snprintf (key, 256, "scrub-duration"); - ret = dict_set_uint32 (*dict, key, scrub_stats->scrub_duration); + ret = dict_set_uint64 (*dict, key, scrub_stats->scrub_duration); if (ret) { gf_msg_debug (this->name, 0, "Failed to set scrub duration" " entry to the dictionary"); @@ -1844,6 +1829,33 @@ br_signer_init (xlator_t *this, br_private_t *priv) } static void +br_free_scrubber_monitor (xlator_t *this, br_private_t *priv) +{ + struct br_monitor *scrub_monitor = &priv->scrub_monitor; + + if (scrub_monitor->timer) { + (void) gf_tw_del_timer (priv->timer_wheel, scrub_monitor->timer); + + GF_FREE (scrub_monitor->timer); + scrub_monitor->timer = NULL; + } + + (void) gf_thread_cleanup_xint (scrub_monitor->thread); + + /* Clean up cond and mutex variables */ + pthread_mutex_destroy (&scrub_monitor->mutex); + pthread_cond_destroy (&scrub_monitor->cond); + + pthread_mutex_destroy (&scrub_monitor->wakelock); + pthread_cond_destroy (&scrub_monitor->wakecond); + + pthread_mutex_destroy (&scrub_monitor->donelock); + pthread_cond_destroy (&scrub_monitor->donecond); + + LOCK_DESTROY (&scrub_monitor->lock); +} + +static void br_free_children (xlator_t *this, br_private_t *priv, int count) { br_child_t *child = NULL; @@ -1878,7 +1890,6 @@ br_init_children (xlator_t *this, br_private_t *priv) LOCK_INIT (&child->lock); child->witnessed = 0; - br_set_scrub_state (child, BR_SCRUB_STATE_INACTIVE); br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED); child->this = this; @@ -1999,6 +2010,9 @@ fini (xlator_t *this) if (!priv->iamscrubber) br_fini_signer (this, priv); + else + (void) br_free_scrubber_monitor (this, priv); + br_free_children (this, priv, priv->child_count); this->private = NULL; @@ -2008,26 +2022,23 @@ fini (xlator_t *this) } static void -br_reconfigure_child (xlator_t *this, br_child_t *child) +br_reconfigure_monitor (xlator_t *this) { int32_t ret = 0; - ret = br_scrub_state_machine (this, child); + ret = br_scrub_state_machine (this); if (ret) { gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_RESCHEDULE_SCRUBBER_FAILED, - "Could not reschedule scrubber for brick: %s. Scubbing " - "will continue according to old frequency.", - child->brick_path); + "Could not reschedule scrubber for the volume. Scrubbing " + "will continue according to old frequency."); } } static int br_reconfigure_scrubber (xlator_t *this, dict_t *options) { - int i = 0; int32_t ret = -1; - br_child_t *child = NULL; br_private_t *priv = NULL; priv = this->private; @@ -2042,32 +2053,11 @@ br_reconfigure_scrubber (xlator_t *this, dict_t *options) goto err; /* change state for all _up_ subvolume(s) */ - for (; i < priv->child_count; i++) { - child = &priv->children[i]; - - LOCK (&child->lock); - { - if (_br_child_failed_conn (child)) { - gf_msg (this->name, GF_LOG_INFO, - 0, BRB_MSG_BRICK_INFO, - "Scrubber for brick [%s] failed " - "initialization, rescheduling is " - "skipped", child->brick_path); - goto unblock; - } - - if (_br_is_child_connected (child)) - br_reconfigure_child (this, child); - - /** - * for the rest.. either the child is in initialization - * phase or is disconnected. either way, updated values - * would be reflected on successful connection. - */ - } - unblock: - UNLOCK (&child->lock); + pthread_mutex_lock (&priv->lock); + { + br_reconfigure_monitor (this); } + pthread_mutex_unlock (&priv->lock); err: return ret; diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h index 04336e641d0..b3d1569ba54 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot.h +++ b/xlators/features/bit-rot/src/bitd/bit-rot.h @@ -26,6 +26,7 @@ #include "bit-rot-common.h" #include "bit-rot-stub-mem-types.h" +#include "bit-rot-scrub-status.h" #include <openssl/sha.h> @@ -63,18 +64,6 @@ struct br_scanfs { unsigned int entries; struct list_head queued; struct list_head ready; - - /* scheduler */ - uint32_t boot; - gf_boolean_t kick; - gf_boolean_t over; - - br_scrub_state_t state; /* current scrub state */ - - pthread_mutex_t wakelock; - pthread_cond_t wakecond; - - struct gf_tw_timer_list *timer; }; /* just need three states to track child status */ @@ -111,6 +100,8 @@ struct br_child { struct timeval tv; struct br_scanfs fsscan; /* per subvolume FS scanner */ + + gf_boolean_t active_scrubbing; /* Actively scrubbing or not */ }; typedef struct br_child br_child_t; @@ -152,27 +143,42 @@ struct br_scrubber { struct list_head scrublist; }; -typedef struct br_obj_n_workers br_obj_n_workers_t; +struct br_monitor { + gf_lock_t lock; + pthread_t thread; /* Monitor thread */ -typedef struct br_private br_private_t; + gf_boolean_t inited; + pthread_mutex_t mutex; + pthread_cond_t cond; /* Thread starts and will be waiting on cond. + First child which is up wakes this up */ -typedef void (*br_scrubbed_file_update) (br_private_t *priv); + xlator_t *this; + /* scheduler */ + uint32_t boot; -struct br_scrub_stats { - uint32_t scrubbed_files; /* Total number of scrubbed file */ + int32_t active_child_count; /* Number of children currently scrubbing */ + gf_boolean_t kick; /* This variable tracks the scrubber is + * kicked or not. Both 'kick' and + * 'active_child_count' uses the same pair + * of mutex-cond variable, i.e, wakelock and + * wakecond. */ - uint32_t unsigned_files; /* Total number of unsigned file */ + pthread_mutex_t wakelock; + pthread_cond_t wakecond; - uint32_t scrub_duration; /* Duration of last scrub */ + gf_boolean_t done; + pthread_mutex_t donelock; + pthread_cond_t donecond; - char last_scrub_time[1024]; /*last scrub completion time */ + struct gf_tw_timer_list *timer; + br_scrub_state_t state; /* current scrub state */ +}; - struct timeval scrub_start_tv; /* Scrubbing starting time*/ +typedef struct br_obj_n_workers br_obj_n_workers_t; - struct timeval scrub_end_tv; /* Scrubbing finishing time */ +typedef struct br_private br_private_t; - pthread_mutex_t lock; -}; +typedef void (*br_scrubbed_file_update) (br_private_t *priv); struct br_private { pthread_mutex_t lock; @@ -209,6 +215,8 @@ struct br_private { struct br_scrub_stats scrub_stat; /* statistics of scrub*/ struct br_scrubber fsscrub; /* scrubbers for this subvolume */ + + struct br_monitor scrub_monitor; /* scrubber monitor */ }; struct br_object { @@ -228,7 +236,7 @@ struct br_object { }; typedef struct br_object br_object_t; -typedef int32_t (br_scrub_ssm_call) (xlator_t *, br_child_t *); +typedef int32_t (br_scrub_ssm_call) (xlator_t *); void br_log_object (xlator_t *, char *, uuid_t, int32_t); @@ -259,6 +267,12 @@ _br_is_child_connected (br_child_t *child) } static inline int +_br_is_child_scrub_active (br_child_t *child) +{ + return child->active_scrubbing; +} + +static inline int _br_child_failed_conn (br_child_t *child) { return (child->c_state == BR_CHILD_STATE_CONNFAILED); @@ -272,10 +286,10 @@ _br_child_witnessed_connection (br_child_t *child) /* scrub state */ static inline void -_br_child_set_scrub_state (br_child_t *child, br_scrub_state_t state) +_br_monitor_set_scrub_state (struct br_monitor *scrub_monitor, + br_scrub_state_t state) { - struct br_scanfs *fsscan = &child->fsscan; - fsscan->state = state; + scrub_monitor->state = state; } static inline br_scrub_event_t |