geo-rep: retry in case of ENOENT errors in entry creations

Change-Id: I8961633a7371c941a3feee44c949d5c934eca998
Original-Author: Venky Shankar <vshankar@redhat.com>
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 847839
Reviewed-on: http://review.gluster.org/5933
Reviewed-by: Venky Shankar <vshankar@redhat.com>
Tested-by: Venky Shankar <vshankar@redhat.com>

2 files changed, 13 insertions, 6 deletions

diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py
index 128ea398000..2583a03cad6 100644
--- a/geo-replication/syncdaemon/resource.py
+++ b/geo-replication/syncdaemon/resource.py
@@ -12,7 +12,7 @@ import logging
 import tempfile
 import threading
 import subprocess
-from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR, ENOTEMPTY
+from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR, ENOTEMPTY, ESTALE, EINVAL
 from select import error as SelectError
 
 from gconf import gconf
@@ -532,7 +532,7 @@ class Server(object):
                 else:
                     errno_wrap(os.rename, [entry, en], [ENOENT, EEXIST])
             if blob:
-                errno_wrap(Xattr.lsetxattr_l, [pg, 'glusterfs.gfid.newfile', blob], [ENOENT, EEXIST])
+                errno_wrap(Xattr.lsetxattr_l, [pg, 'glusterfs.gfid.newfile', blob], [EEXIST], [ENOENT, ESTALE, EINVAL])
 
     @classmethod
     def changelog_register(cls, cl_brick, cl_dir, cl_log, cl_level, retries = 0):
diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py
index 2655dd9835e..348eb38c1d0 100644
--- a/geo-replication/syncdaemon/syncdutils.py
+++ b/geo-replication/syncdaemon/syncdutils.py
@@ -35,6 +35,7 @@ except ImportError:
 
 # auxillary gfid based access prefix
 _CL_AUX_GFID_PFX = ".gfid/"
+GF_OP_RETRIES = 20
 
 def escape(s):
     """the chosen flavor of string escaping, used all over
@@ -405,10 +406,11 @@ def md5hex(s):
 def selfkill(sig=SIGTERM):
     os.kill(os.getpid(), sig)
 
-def errno_wrap(call, arg=[], errnos=[]):
+def errno_wrap(call, arg=[], errnos=[], retry_errnos=[ESTALE]):
     """ wrapper around calls resilient to errnos.
-    retry in case of ESTALE
+    retry in case of ESTALE by default.
     """
+    nr_tries = 0
     while True:
         try:
             return call(*arg)
@@ -416,9 +418,14 @@ def errno_wrap(call, arg=[], errnos=[]):
             ex = sys.exc_info()[1]
             if ex.errno in errnos:
                 return ex.errno
-            if not ex.errno == ESTALE:
+            if not ex.errno in retry_errnos:
                 raise
-            time.sleep(0.5)  # retry the call
+            nr_tries += 1
+            if nr_tries == GF_OP_RETRIES:
+                # probably a screwed state, cannot do much...
+                logging.warn('reached maximum retries (%s)...' % repr(arg))
+                return
+            time.sleep(0.250)  # retry the call
 
 def lstat(e):
     try: