summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorCsaba Henk <csaba@redhat.com>2020-05-08 23:01:04 +0200
committerRinku Kothiya <rkothiya@redhat.com>2020-05-28 07:29:02 +0000
commit2ae5b28030b08348ec5e4c08ada8600943db0366 (patch)
tree9e58fd0c9e18d493e4220bae7b0d921fa80cf1cd
parent984a8dd8fa3e371fa1336730d6dc0041ceef877c (diff)
fuse: occasional logging for fuse device 'weird' write errors
This change is a followup to I510158843e4b1d482bdc496c2e97b1860dc1ba93. In referred change we pushed log messages about 'weird' write errors to fuse device out of sight, by reporting them at Debug loglevel instead of Error (where 'weird' means errno is not POSIX compliant but having meaningful semantics for FUSE protocol). This solved the issue of spurious error reporting. And so far so good: these messages don't indicate an error condition by themselves. However, when they come in high repetitions, that indicates a suboptimal condition which should be reported.[1] Therefore now we shall emit a Warning if a certain errno occurs a certain number of times[2] as the outcome of a write to the fuse device. ___ [1] typically ENOENTs and ENOTDIRs accumulate when glusterfs' inode invalidation lags behind the kernel's internal inode garbage collection (in this case above errnos mean that the inode which we requested to be invalidated is not found in kernel). This can be mitigated with the invalidate-limit command line / mount option, cf. bz#1732717. [2] 256, as of the current implementation. Change-Id: I8cc7fe104da43a88875f93b0db49d5677cc16045 Updates: #1000 Signed-off-by: Csaba Henk <csaba@redhat.com> (cherry picked from commit c1baf3c68b87584aea5389af958326f6ed01d7ec)
-rw-r--r--xlators/mount/fuse/src/fuse-bridge.c36
-rw-r--r--xlators/mount/fuse/src/fuse-bridge.h18
2 files changed, 53 insertions, 1 deletions
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 7949241ab0f..72a36fefa21 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -225,14 +225,30 @@ check_and_dump_fuse_W(fuse_private_t *priv, struct iovec *iov_out, int count,
if (res == -1) {
const char *errdesc = NULL;
gf_loglevel_t loglevel = GF_LOG_ERROR;
+ gf_boolean_t errno_degraded = _gf_false;
+ gf_boolean_t errno_promoted = _gf_false;
+
+#define ACCOUNT_ERRNO(eno) \
+ do { \
+ if (errno_degraded) { \
+ pthread_mutex_lock(&priv->fusedev_errno_cnt_mutex); \
+ { \
+ if (!++priv->fusedev_errno_cnt[FUSEDEV_##eno]) \
+ errno_promoted = _gf_true; \
+ } \
+ pthread_mutex_unlock(&priv->fusedev_errno_cnt_mutex); \
+ } \
+ } while (0)
/* If caller masked the errno, then it
* does not indicate an error at the application
* level, so we degrade the log severity to DEBUG.
*/
if (errnomask && errno < ERRNOMASK_MAX &&
- GET_ERRNO_MASK(errnomask, errno))
+ GET_ERRNO_MASK(errnomask, errno)) {
loglevel = GF_LOG_DEBUG;
+ errno_degraded = _gf_true;
+ }
switch (errno) {
/* The listed errnos are FUSE status indicators,
@@ -242,33 +258,43 @@ check_and_dump_fuse_W(fuse_private_t *priv, struct iovec *iov_out, int count,
*/
case ENOENT:
errdesc = "ENOENT";
+ ACCOUNT_ERRNO(ENOENT);
break;
case ENOTDIR:
errdesc = "ENOTDIR";
+ ACCOUNT_ERRNO(ENOTDIR);
break;
case ENODEV:
errdesc = "ENODEV";
+ ACCOUNT_ERRNO(ENODEV);
break;
case EPERM:
errdesc = "EPERM";
+ ACCOUNT_ERRNO(EPERM);
break;
case ENOMEM:
errdesc = "ENOMEM";
+ ACCOUNT_ERRNO(ENOMEM);
break;
case ENOTCONN:
errdesc = "ENOTCONN";
+ ACCOUNT_ERRNO(ENOTCONN);
break;
case ECONNREFUSED:
errdesc = "ECONNREFUSED";
+ ACCOUNT_ERRNO(ECONNREFUSED);
break;
case EOVERFLOW:
errdesc = "EOVERFLOW";
+ ACCOUNT_ERRNO(EOVERFLOW);
break;
case EBUSY:
errdesc = "EBUSY";
+ ACCOUNT_ERRNO(EBUSY);
break;
case ENOTEMPTY:
errdesc = "ENOTEMPTY";
+ ACCOUNT_ERRNO(ENOTEMPTY);
break;
default:
errdesc = strerror(errno);
@@ -276,7 +302,13 @@ check_and_dump_fuse_W(fuse_private_t *priv, struct iovec *iov_out, int count,
gf_log_callingfn("glusterfs-fuse", loglevel,
"writing to fuse device failed: %s", errdesc);
+ if (errno_promoted)
+ gf_log("glusterfs-fuse", GF_LOG_WARNING,
+ "writing to fuse device yielded %s %d times", errdesc,
+ UINT8_MAX + 1);
return errno;
+
+#undef ACCOUNT_ERRNO
}
fouh = iov_out[0].iov_base;
@@ -6671,6 +6703,8 @@ init(xlator_t *this_xl)
INIT_LIST_HEAD(&priv->interrupt_list);
pthread_mutex_init(&priv->interrupt_mutex, NULL);
+ pthread_mutex_init(&priv->fusedev_errno_cnt_mutex, NULL);
+
/* get options from option dictionary */
ret = dict_get_str(options, ZR_MOUNTPOINT_OPT, &value_string);
if (ret == -1 || value_string == NULL) {
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index f4c6ad8f2fb..2371692ec0b 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -78,6 +78,20 @@ typedef struct fuse_in_header fuse_in_header_t;
typedef void(fuse_handler_t)(xlator_t *this, fuse_in_header_t *finh, void *msg,
struct iobuf *iobuf);
+enum fusedev_errno {
+ FUSEDEV_ENOENT,
+ FUSEDEV_ENOTDIR,
+ FUSEDEV_ENODEV,
+ FUSEDEV_EPERM,
+ FUSEDEV_ENOMEM,
+ FUSEDEV_ENOTCONN,
+ FUSEDEV_ECONNREFUSED,
+ FUSEDEV_EOVERFLOW,
+ FUSEDEV_EBUSY,
+ FUSEDEV_ENOTEMPTY,
+ FUSEDEV_EMAXPLUS
+};
+
struct fuse_private {
int fd;
uint32_t proto_minor;
@@ -193,6 +207,10 @@ struct fuse_private {
uint32_t lru_limit;
uint32_t invalidate_limit;
uint32_t fuse_dev_eperm_ratelimit_ns;
+
+ /* counters for fusdev errnos */
+ uint8_t fusedev_errno_cnt[FUSEDEV_EMAXPLUS];
+ pthread_mutex_t fusedev_errno_cnt_mutex;
};
typedef struct fuse_private fuse_private_t;