From 9c7a0aaf73993380c70570f99023b83064a85ddf Mon Sep 17 00:00:00 2001 From: Kotresh HR Date: Tue, 10 Jan 2017 00:30:42 -0500 Subject: geo-rep: Handle directory sync failure as hard error If directory creation is failed, return immediately before further processing. Allowing it to further process will fail the entire directory tree syncing to slave. Hence master will log and raise exception if it's directory failure. Earlier, master used to log the failure and proceed. > BUG: 1411607 > Signed-off-by: Kotresh HR > Reviewed-on: http://review.gluster.org/16364 > Smoke: Gluster Build System > NetBSD-regression: NetBSD Build System > CentOS-regression: Gluster Build System > Reviewed-by: Aravinda VK Change-Id: Iba2a8b5d3d0092e7a9c8a3c2cdf9e6e29c73ddf0 BUG: 1441933 Signed-off-by: Kotresh HR Reviewed-on: https://review.gluster.org/17051 Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Reviewed-by: Aravinda VK --- geo-replication/syncdaemon/master.py | 3 +++ geo-replication/syncdaemon/resource.py | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py index 6437dcc4295..f7667cf0cf6 100644 --- a/geo-replication/syncdaemon/master.py +++ b/geo-replication/syncdaemon/master.py @@ -783,6 +783,9 @@ class GMasterChangelogMixin(GMasterCommon): num_failures += 1 logging.error('%s FAILED: %s' % (log_prefix, repr(failure))) + if failure[0]['op'] == 'MKDIR': + raise GsyncdError("The above directory failed to sync." + " Please fix it to proceed further.") self.status.inc_value("failures", num_failures) diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py index ecb94d56c7f..2e607799c57 100644 --- a/geo-replication/syncdaemon/resource.py +++ b/geo-replication/syncdaemon/resource.py @@ -630,16 +630,19 @@ class Server(object): # We do this for failing fops on Slave # Master should be logging this if cmd_ret is None: - return + return False if cmd_ret == EEXIST: disk_gfid = cls.gfid_mnt(e['entry']) - if isinstance(disk_gfid, basestring): - if e['gfid'] != disk_gfid: - failures.append((e, cmd_ret, disk_gfid)) + if isinstance(disk_gfid, basestring) and e['gfid'] != disk_gfid: + failures.append((e, cmd_ret, disk_gfid)) + else: + return False else: failures.append((e, cmd_ret)) + return True + failures = [] def matching_disk_gfid(gfid, entry): @@ -809,7 +812,15 @@ class Server(object): [pg, 'glusterfs.gfid.newfile', blob], [EEXIST, ENOENT], [ESTALE, EINVAL]) - collect_failure(e, cmd_ret) + failed = collect_failure(e, cmd_ret) + + # If directory creation is failed, return immediately before + # further processing. Allowing it to further process will + # cause the entire directory tree to fail syncing to slave. + # Hence master will log and raise exception if it's + # directory failure. + if failed and op == 'MKDIR': + return failures # If UID/GID is different than zero that means we are trying # create Entry with different UID/GID. Create Entry with @@ -818,7 +829,7 @@ class Server(object): path = os.path.join(pfx, gfid) cmd_ret = errno_wrap(os.chown, [path, uid, gid], [ENOENT], [ESTALE, EINVAL]) - collect_failure(e, cmd_ret) + collect_failure(e, cmd_ret) return failures -- cgit