syncdaemon: fortify handling of failed xtime queries

Signed-off-by: Kaushik BV <kaushikbv@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

2 files changed, 19 insertions, 6 deletions

diff --git a/xlators/features/marker/utils/syncdaemon/master.py b/xlators/features/marker/utils/syncdaemon/master.py
index 0f12f96cf..a275f55fb 100644
--- a/xlators/features/marker/utils/syncdaemon/master.py
+++ b/xlators/features/marker/utils/syncdaemon/master.py
@@ -38,7 +38,9 @@ class GMaster(object):
         if not 'create' in opts:
             opts['create'] = rsc == self.master
         xt = rsc.server.xtime(path, self.uuid)
-        if (isinstance(xt, int) or xt < self.volmark) and opts['create']:
+        if isinstance(xt, int) and xt != ENODATA:
+            return xt
+        if (xt == ENODATA or xt < self.volmark) and opts['create']:
             t = time.time()
             sec = int(t)
             nsec = int((t - sec) * 1000000)
@@ -67,6 +69,10 @@ class GMaster(object):
             self.jobtab[path] = []
         self.jobtab[path].append((label, a, lambda : job(*a, **kw)))
 
+    def add_failjob(self, path, label):
+        logging.debug('salvaged: ' + label)
+        self.add_job(path, label, lambda: False)
+
     def wait(self, path, *args):
         jobs = self.jobtab.pop(path, [])
         succeed = True
@@ -96,14 +102,21 @@ class GMaster(object):
         logging.debug("entering " + path)
         if not xtl:
             xtl = self.xtime(path)
+            if isinstance(xtl, int):
+                self.add_failjob(path, 'no-local-node')
+                return
         xtr0 = self.xtime(path, self.slave)
         if isinstance(xtr0, int):
+            if xtr0 != ENOENT:
+                self.slave.server.purge(path)
+            try:
+                self.slave.server.mkdir(path)
+            except OSError:
+                self.add_failjob(path, 'no-remote-node')
+                return
             xtr = URXTIME
         else:
             xtr = xtr0
-        if xtr0 == ENOENT:
-            self.slave.server.mkdir(path)
-        else:
             if xtr > xtl:
                 raise RuntimeError("timestamp corruption for " + path)
             if xtl == xtr:
@@ -152,7 +165,7 @@ class GMaster(object):
                 ex = sys.exc_info()[1]
                 if ex.errno == ENOENT:
                     logging.warn("salvaged ENOENT for" + e)
-                    self.add_job(blame, 'salvage', lambda: False)
+                    self.add_failjob(blame, 'by-indulgently')
                     return False
                 else:
                     raise
diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/xlators/features/marker/utils/syncdaemon/resource.py
index 599267824..a56b6caa0 100644
--- a/xlators/features/marker/utils/syncdaemon/resource.py
+++ b/xlators/features/marker/utils/syncdaemon/resource.py
@@ -157,7 +157,7 @@ class Server(object):
             return struct.unpack('!II', Xattr.lgetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), 8))
         except OSError:
             ex = sys.exc_info()[1]
-            if ex.errno in (ENOENT, ENODATA):
+            if ex.errno in (ENOENT, ENODATA, ENOTDIR):
                 return ex.errno
             else:
                 raise