From 52ce8fc0a8a8b87afe3e77d5aeee22baa7f216f8 Mon Sep 17 00:00:00 2001 From: Amar Tumballi Date: Mon, 16 Sep 2013 14:02:25 +0530 Subject: geo-rep: retry in case of ENOENT errors in entry creations Change-Id: I8961633a7371c941a3feee44c949d5c934eca998 Original-Author: Venky Shankar Signed-off-by: Amar Tumballi BUG: 847839 Reviewed-on: http://review.gluster.org/5933 Reviewed-by: Venky Shankar Tested-by: Venky Shankar --- geo-replication/syncdaemon/resource.py | 4 ++-- geo-replication/syncdaemon/syncdutils.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) (limited to 'geo-replication') diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py index 128ea3980..2583a03ca 100644 --- a/geo-replication/syncdaemon/resource.py +++ b/geo-replication/syncdaemon/resource.py @@ -12,7 +12,7 @@ import logging import tempfile import threading import subprocess -from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR, ENOTEMPTY +from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR, ENOTEMPTY, ESTALE, EINVAL from select import error as SelectError from gconf import gconf @@ -532,7 +532,7 @@ class Server(object): else: errno_wrap(os.rename, [entry, en], [ENOENT, EEXIST]) if blob: - errno_wrap(Xattr.lsetxattr_l, [pg, 'glusterfs.gfid.newfile', blob], [ENOENT, EEXIST]) + errno_wrap(Xattr.lsetxattr_l, [pg, 'glusterfs.gfid.newfile', blob], [EEXIST], [ENOENT, ESTALE, EINVAL]) @classmethod def changelog_register(cls, cl_brick, cl_dir, cl_log, cl_level, retries = 0): diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py index 2655dd983..348eb38c1 100644 --- a/geo-replication/syncdaemon/syncdutils.py +++ b/geo-replication/syncdaemon/syncdutils.py @@ -35,6 +35,7 @@ except ImportError: # auxillary gfid based access prefix _CL_AUX_GFID_PFX = ".gfid/" +GF_OP_RETRIES = 20 def escape(s): """the chosen flavor of string escaping, used all over @@ -405,10 +406,11 @@ def md5hex(s): def selfkill(sig=SIGTERM): os.kill(os.getpid(), sig) -def errno_wrap(call, arg=[], errnos=[]): +def errno_wrap(call, arg=[], errnos=[], retry_errnos=[ESTALE]): """ wrapper around calls resilient to errnos. - retry in case of ESTALE + retry in case of ESTALE by default. """ + nr_tries = 0 while True: try: return call(*arg) @@ -416,9 +418,14 @@ def errno_wrap(call, arg=[], errnos=[]): ex = sys.exc_info()[1] if ex.errno in errnos: return ex.errno - if not ex.errno == ESTALE: + if not ex.errno in retry_errnos: raise - time.sleep(0.5) # retry the call + nr_tries += 1 + if nr_tries == GF_OP_RETRIES: + # probably a screwed state, cannot do much... + logging.warn('reached maximum retries (%s)...' % repr(arg)) + return + time.sleep(0.250) # retry the call def lstat(e): try: -- cgit