diff options
| -rw-r--r-- | geo-replication/syncdaemon/resource.py | 4 | ||||
| -rw-r--r-- | geo-replication/syncdaemon/syncdutils.py | 15 |
2 files changed, 13 insertions, 6 deletions
diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py index 128ea398000..2583a03cad6 100644 --- a/geo-replication/syncdaemon/resource.py +++ b/geo-replication/syncdaemon/resource.py @@ -12,7 +12,7 @@ import logging import tempfile import threading import subprocess -from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR, ENOTEMPTY +from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR, ENOTEMPTY, ESTALE, EINVAL from select import error as SelectError from gconf import gconf @@ -532,7 +532,7 @@ class Server(object): else: errno_wrap(os.rename, [entry, en], [ENOENT, EEXIST]) if blob: - errno_wrap(Xattr.lsetxattr_l, [pg, 'glusterfs.gfid.newfile', blob], [ENOENT, EEXIST]) + errno_wrap(Xattr.lsetxattr_l, [pg, 'glusterfs.gfid.newfile', blob], [EEXIST], [ENOENT, ESTALE, EINVAL]) @classmethod def changelog_register(cls, cl_brick, cl_dir, cl_log, cl_level, retries = 0): diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py index 2655dd9835e..348eb38c1d0 100644 --- a/geo-replication/syncdaemon/syncdutils.py +++ b/geo-replication/syncdaemon/syncdutils.py @@ -35,6 +35,7 @@ except ImportError: # auxillary gfid based access prefix _CL_AUX_GFID_PFX = ".gfid/" +GF_OP_RETRIES = 20 def escape(s): """the chosen flavor of string escaping, used all over @@ -405,10 +406,11 @@ def md5hex(s): def selfkill(sig=SIGTERM): os.kill(os.getpid(), sig) -def errno_wrap(call, arg=[], errnos=[]): +def errno_wrap(call, arg=[], errnos=[], retry_errnos=[ESTALE]): """ wrapper around calls resilient to errnos. - retry in case of ESTALE + retry in case of ESTALE by default. """ + nr_tries = 0 while True: try: return call(*arg) @@ -416,9 +418,14 @@ def errno_wrap(call, arg=[], errnos=[]): ex = sys.exc_info()[1] if ex.errno in errnos: return ex.errno - if not ex.errno == ESTALE: + if not ex.errno in retry_errnos: raise - time.sleep(0.5) # retry the call + nr_tries += 1 + if nr_tries == GF_OP_RETRIES: + # probably a screwed state, cannot do much... + logging.warn('reached maximum retries (%s)...' % repr(arg)) + return + time.sleep(0.250) # retry the call def lstat(e): try: |