From e48c5f8a84a1ec27492bcede0a028755a440be45 Mon Sep 17 00:00:00 2001 From: Aravinda VK Date: Wed, 18 Dec 2013 00:02:58 +0530 Subject: geo-rep: optimizing update stime after directory synchronization Since xsync crawl generates new changelog when number of entries reaches 8K or when directory is reached. If a directory has number of files less than 8K then respective changelog file will have less entries. Since xsync generated changelog files processed one after the other, so syncjobs are underutilized. hence low bandwidth utilization. With this patch, changelog will be generated for 8K entries only, but stime will be accumulated. Multiple dirs stime will be updated together since the generated changelog will have entries accross the dirs. Change-Id: Ib0b40962a070f855f47f887d0840e412fb7928e1 Signed-off-by: Aravinda VK Reviewed-on: http://review.gluster.org/6744 Tested-by: Gluster Build System Reviewed-by: Venky Shankar Tested-by: Venky Shankar --- geo-replication/syncdaemon/master.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'geo-replication') diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py index 721fe18bd..662d2ba74 100644 --- a/geo-replication/syncdaemon/master.py +++ b/geo-replication/syncdaemon/master.py @@ -1091,6 +1091,7 @@ class GMasterXsyncMixin(GMasterChangelogMixin): def register(self): self.counter = 0 self.comlist = [] + self.stimes = [] self.sleep_interval = 60 self.tempdir = self.setup_working_dir()[0] self.tempdir = os.path.join(self.tempdir, 'xsync') @@ -1169,10 +1170,15 @@ class GMasterXsyncMixin(GMasterChangelogMixin): if last: self.put('finale', None) - def sync_done(self, stime=None, last=False): + def sync_done(self, stime=[], last=False): self.sync_xsync(last) if stime: - self.sync_stime(stime, last) + # Send last as True only for last stime entry + for st in stime[:-1]: + self.sync_stime(st, False) + + if stime and stime[-1]: + self.sync_stime(stime[-1], last) def Xcrawl(self, path='.', xtr_root=None): """ @@ -1204,7 +1210,7 @@ class GMasterXsyncMixin(GMasterChangelogMixin): xtr = max(xtr, xtr_root) if not self.need_sync(path, xtl, xtr): if path == '.': - self.sync_done((path, xtl), True) + self.sync_done([(path, xtl)], True) return self.xtime_reversion_hook(path, xtl, xtr) logging.debug("entering " + path) @@ -1232,11 +1238,12 @@ class GMasterXsyncMixin(GMasterChangelogMixin): mo = st.st_mode self.counter += 1 if self.counter == self.XSYNC_MAX_ENTRIES: - self.sync_done() + self.sync_done(self.stimes, False) + self.stimes = [] if stat.S_ISDIR(mo): self.write_entry_change("E", [gfid, 'MKDIR', str(mo), str(st.st_uid), str(st.st_gid), escape(os.path.join(pargfid, bname))]) self.Xcrawl(e, xtr_root) - self.sync_done((e, xte), False) + self.stimes.append((e, xte)) elif stat.S_ISLNK(mo): self.write_entry_change("E", [gfid, 'SYMLINK', escape(os.path.join(pargfid, bname))]) elif stat.S_ISREG(mo): @@ -1250,7 +1257,8 @@ class GMasterXsyncMixin(GMasterChangelogMixin): self.write_entry_change("E", [gfid, 'LINK', escape(os.path.join(pargfid, bname))]) self.write_entry_change("D", [gfid]) if path == '.': - self.sync_done((path, xtl), True) + self.stimes.append((path, xtl)) + self.sync_done(self.stimes, True) class BoxClosedErr(Exception): pass -- cgit