diff options
| author | Aravinda VK <avishwan@redhat.com> | 2013-12-18 00:02:58 +0530 |
|---|---|---|
| committer | Vijay Bellur <vbellur@redhat.com> | 2014-02-02 20:25:27 -0800 |
| commit | 540de26e2ef7c45eb0ef6c049844a864542bd38a (patch) | |
| tree | 0b51f4ffd3068250879cd93fdd2d12c1f5fa0205 | |
| parent | 0f018e33ec8510ef0afc7daa9f12b7e21c8c03dc (diff) | |
geo-rep: optimizing update stime after directory synchronization
Since xsync crawl generates new changelog when number of entries
reaches 8K or when directory is reached. If a directory has number
of files less than 8K then respective changelog file will have less
entries. Since xsync generated changelog files processed one after
the other, so syncjobs are underutilized. hence low bandwidth
utilization.
With this patch, changelog will be generated for 8K entries only,
but stime will be accumulated. Multiple dirs stime will be
updated together since the generated changelog will have entries
accross the dirs.
BUG: 1036539
Change-Id: Ic4b9da6d8c1f11dd9ef7c13bcc7df80efe2c5caf
Signed-off-by: Aravinda VK <avishwan@redhat.com>
Reviewed-on: http://review.gluster.org/6744
Reviewed-on: http://review.gluster.org/6847
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Venky Shankar <vshankar@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>
| -rw-r--r-- | geo-replication/syncdaemon/master.py | 20 |
1 files changed, 14 insertions, 6 deletions
diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py index 721fe18bd18..662d2ba7481 100644 --- a/geo-replication/syncdaemon/master.py +++ b/geo-replication/syncdaemon/master.py @@ -1091,6 +1091,7 @@ class GMasterXsyncMixin(GMasterChangelogMixin): def register(self): self.counter = 0 self.comlist = [] + self.stimes = [] self.sleep_interval = 60 self.tempdir = self.setup_working_dir()[0] self.tempdir = os.path.join(self.tempdir, 'xsync') @@ -1169,10 +1170,15 @@ class GMasterXsyncMixin(GMasterChangelogMixin): if last: self.put('finale', None) - def sync_done(self, stime=None, last=False): + def sync_done(self, stime=[], last=False): self.sync_xsync(last) if stime: - self.sync_stime(stime, last) + # Send last as True only for last stime entry + for st in stime[:-1]: + self.sync_stime(st, False) + + if stime and stime[-1]: + self.sync_stime(stime[-1], last) def Xcrawl(self, path='.', xtr_root=None): """ @@ -1204,7 +1210,7 @@ class GMasterXsyncMixin(GMasterChangelogMixin): xtr = max(xtr, xtr_root) if not self.need_sync(path, xtl, xtr): if path == '.': - self.sync_done((path, xtl), True) + self.sync_done([(path, xtl)], True) return self.xtime_reversion_hook(path, xtl, xtr) logging.debug("entering " + path) @@ -1232,11 +1238,12 @@ class GMasterXsyncMixin(GMasterChangelogMixin): mo = st.st_mode self.counter += 1 if self.counter == self.XSYNC_MAX_ENTRIES: - self.sync_done() + self.sync_done(self.stimes, False) + self.stimes = [] if stat.S_ISDIR(mo): self.write_entry_change("E", [gfid, 'MKDIR', str(mo), str(st.st_uid), str(st.st_gid), escape(os.path.join(pargfid, bname))]) self.Xcrawl(e, xtr_root) - self.sync_done((e, xte), False) + self.stimes.append((e, xte)) elif stat.S_ISLNK(mo): self.write_entry_change("E", [gfid, 'SYMLINK', escape(os.path.join(pargfid, bname))]) elif stat.S_ISREG(mo): @@ -1250,7 +1257,8 @@ class GMasterXsyncMixin(GMasterChangelogMixin): self.write_entry_change("E", [gfid, 'LINK', escape(os.path.join(pargfid, bname))]) self.write_entry_change("D", [gfid]) if path == '.': - self.sync_done((path, xtl), True) + self.stimes.append((path, xtl)) + self.sync_done(self.stimes, True) class BoxClosedErr(Exception): pass |