From c6d7ddc4bcdefbe7e30946c5c7eb26e74ad0ff0e Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 30 Oct 2013 16:13:50 +0530 Subject: Improve logging and raising DiskFileNoSpace This commit only improves logging whenever ENOSPC (No space on disk) or EDQUOT (Quota limit exceeded) is returned by glusterfs Also, added methods to: - get filename from file descriptor - log with rate limit Caveat: Although raising DiskFileNoSpace results in object-server returning HTTPInsufficientStorage[507] correctly, the swift proxy-server invokes "best_response" method that returns [503] to the user. When write-behind translator is turned on in glusterfs, it may set errno to EIO instead of ENOSPC/EDQUOT. This is documented in BZ 986812 BUG: 985862, 985253, 1020724 Change-Id: Ib0c5e41c11a8cdccc2077f71c31d8a23229452bb Signed-off-by: Prashanth Pai Reviewed-on: http://review.gluster.org/6199 Reviewed-by: Luis Pabon Tested-by: Luis Pabon Reviewed-on: http://review.gluster.org/6269 --- gluster/swift/common/fs_utils.py | 102 +++++++++++++++++++++++++++++++++++++-- gluster/swift/common/utils.py | 19 ++++++-- 2 files changed, 113 insertions(+), 8 deletions(-) (limited to 'gluster/swift/common') diff --git a/gluster/swift/common/fs_utils.py b/gluster/swift/common/fs_utils.py index 199984a..1ef68ed 100644 --- a/gluster/swift/common/fs_utils.py +++ b/gluster/swift/common/fs_utils.py @@ -18,12 +18,16 @@ import os import errno import stat import random +import time +from collections import defaultdict +from itertools import repeat import ctypes import os.path as _os_path from eventlet import sleep from swift.common.utils import load_libc_function from gluster.swift.common.exceptions import FileOrDirNotFoundError, \ NotDirectoryError, GlusterFileSystemOSError +from swift.common.exceptions import DiskFileNoSpace os_path = _os_path @@ -37,8 +41,14 @@ def do_write(fd, buf): try: cnt = os.write(fd, buf) except OSError as err: - raise GlusterFileSystemOSError( - err.errno, '%s, os.write("%s", ...)' % (err.strerror, fd)) + filename = get_filename_from_fd(fd) + if err.errno in (errno.ENOSPC, errno.EDQUOT): + do_log_rl("do_write(%d, msg[%d]) failed: %s : %s", + fd, len(buf), err, filename) + raise DiskFileNoSpace() + else: + raise GlusterFileSystemOSError( + err.errno, '%s, os.write("%s", ...)' % (err.strerror, fd)) return cnt @@ -100,6 +110,9 @@ def do_mkdir(path): except OSError as err: if err.errno == errno.EEXIST: logging.warn("fs_utils: os.mkdir - path %s already exists", path) + elif err.errno in (errno.ENOSPC, errno.EDQUOT): + do_log_rl("do_mkdir(%s) failed: %s", path, err) + raise DiskFileNoSpace() else: raise GlusterFileSystemOSError( err.errno, '%s, os.mkdir("%s")' % (err.strerror, path)) @@ -288,5 +301,86 @@ def mkdirs(path): except OSError as err: if err.errno == errno.EEXIST and os.path.isdir(path): return - raise GlusterFileSystemOSError( - err.errno, '%s, os.makedirs("%s")' % (err.strerror, path)) + elif err.errno in (errno.ENOSPC, errno.EDQUOT): + do_log_rl("mkdirs(%s) failed: %s", path, err) + raise DiskFileNoSpace() + else: + raise GlusterFileSystemOSError( + err.errno, '%s, os.makedirs("%s")' % (err.strerror, path)) + + +def get_filename_from_fd(fd, verify=False): + """ + Given the file descriptor, this method attempts to get the filename as it + was when opened. This may not give accurate results in following cases: + - file was renamed/moved/deleted after it was opened + - file has multiple hardlinks + + :param fd: file descriptor of file opened + :param verify: If True, performs additional checks using inode number + """ + filename = None + if isinstance(fd, int): + try: + filename = os.readlink("/proc/self/fd/" + str(fd)) + except OSError: + pass + + if not verify: + return filename + + # If verify = True, we compare st_dev and st_ino of file and fd. + # This involves additional stat and fstat calls. So this is disabled + # by default. + if filename and fd: + s_file = do_stat(filename) + s_fd = do_fstat(fd) + + if s_file and s_fd: + if (s_file.st_ino, s_file.st_dev) == (s_fd.st_ino, s_fd.st_dev): + return filename + + return None + + +def static_var(varname, value): + """Decorator function to create pseudo static variables.""" + def decorate(func): + setattr(func, varname, value) + return func + return decorate + +# Rate limit to emit log message once a second +_DO_LOG_RL_INTERVAL = 1.0 + + +@static_var("counter", defaultdict(int)) +@static_var("last_called", defaultdict(repeat(0.0).next)) +def do_log_rl(msg, *args, **kwargs): + """ + Rate limited logger. + + :param msg: String or message to be logged + :param log_level: Possible values- error, warning, info, debug, critical + """ + log_level = kwargs.get('log_level', "error") + if log_level not in ("error", "warning", "info", "debug", "critical"): + log_level = "error" + + do_log_rl.counter[msg] += 1 # Increment msg counter + interval = time.time() - do_log_rl.last_called[msg] + + if interval >= _DO_LOG_RL_INTERVAL: + # Prefix PID of process and message count to original log msg + emit_msg = "[PID:" + str(os.getpid()) + "]" \ + + "[RateLimitedLog;Count:" + str(do_log_rl.counter[msg]) + "] " \ + + msg + # log_level is a param for do_log_rl and not for logging.* methods + try: + del kwargs['log_level'] + except KeyError: + pass + + getattr(logging, log_level)(emit_msg, *args, **kwargs) # Emit msg + do_log_rl.counter[msg] = 0 # Reset msg counter when message is emitted + do_log_rl.last_called[msg] = time.time() # Reset msg time diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py index 6773836..0b2ad28 100644 --- a/gluster/swift/common/utils.py +++ b/gluster/swift/common/utils.py @@ -22,8 +22,9 @@ from hashlib import md5 from eventlet import sleep import cPickle as pickle from gluster.swift.common.exceptions import GlusterFileSystemIOError +from swift.common.exceptions import DiskFileNoSpace from gluster.swift.common.fs_utils import os_path, do_stat, do_listdir, \ - do_walk, do_rmdir, do_fstat + do_walk, do_rmdir, do_fstat, do_log_rl, get_filename_from_fd from gluster.swift.common import Glusterfs X_CONTENT_TYPE = 'Content-Type' @@ -137,9 +138,19 @@ def write_metadata(path_or_fd, metadata): '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE]) except IOError as err: - raise GlusterFileSystemIOError( - err.errno, - 'xattr.setxattr("%s", %s, metastr)' % (path_or_fd, key)) + if err.errno in (errno.ENOSPC, errno.EDQUOT): + if isinstance(path_or_fd, int): + filename = get_filename_from_fd(path_or_fd) + do_log_rl("write_metadata(%d, metadata) failed: %s : %s", + path_or_fd, err, filename) + else: + do_log_rl("write_metadata(%s, metadata) failed: %s", + path_or_fd, err) + raise DiskFileNoSpace() + else: + raise GlusterFileSystemIOError( + err.errno, + 'xattr.setxattr("%s", %s, metastr)' % (path_or_fd, key)) metastr = metastr[MAX_XATTR_SIZE:] key += 1 -- cgit