diff options
| -rw-r--r-- | geo-replication/syncdaemon/gsyncd.py | 1 | ||||
| -rw-r--r-- | geo-replication/syncdaemon/master.py | 10 | ||||
| -rw-r--r-- | geo-replication/syncdaemon/monitor.py | 17 | ||||
| -rw-r--r-- | geo-replication/syncdaemon/resource.py | 7 |
4 files changed, 30 insertions, 5 deletions
diff --git a/geo-replication/syncdaemon/gsyncd.py b/geo-replication/syncdaemon/gsyncd.py index b3c7e62506e..982b8532f75 100644 --- a/geo-replication/syncdaemon/gsyncd.py +++ b/geo-replication/syncdaemon/gsyncd.py @@ -356,6 +356,7 @@ def main_i(): action='callback', callback=store_local_curry('canon')) op.add_option('--canonicalize-escape-url', dest='url_print', action='callback', callback=store_local_curry('canon_esc')) + op.add_option('--is-coldtier', default=False, action='store_true') tunables = [norm(o.get_opt_string()[2:]) for o in op.option_list diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py index 407e4b29580..b2cfb8531d1 100644 --- a/geo-replication/syncdaemon/master.py +++ b/geo-replication/syncdaemon/master.py @@ -809,6 +809,13 @@ class GMasterChangelogMixin(GMasterCommon): et = e[self.IDX_START:self.IDX_END] # entry type ec = e[self.IDX_END:].split(' ') # rest of the bits + # skip ENTRY operation if cold tier brick + if self.name == 'live_changelog': + if boolify(gconf.is_coldtier) and et == self.TYPE_ENTRY: + logging.debug('skip ENTRY op: %s if cold tier brick' + % (ec[self.POS_TYPE])) + continue + if et == self.TYPE_ENTRY: # extract information according to the type of # the entry operation. create(), mkdir() and mknod() @@ -1120,6 +1127,7 @@ class GMasterChangelogMixin(GMasterCommon): self.changelog_done_func = self.changelog_agent.done self.processed_changelogs_dir = os.path.join(self.setup_working_dir(), ".processed") + self.name = "live_changelog" self.status = status @@ -1132,6 +1140,7 @@ class GMasterChangeloghistoryMixin(GMasterChangelogMixin): self.history_turns = 0 self.processed_changelogs_dir = os.path.join(self.setup_working_dir(), ".history/.processed") + self.name = "history_changelog" self.status = status def crawl(self): @@ -1226,6 +1235,7 @@ class GMasterXsyncMixin(GMasterChangelogMixin): self.tempdir = self.setup_working_dir() self.tempdir = os.path.join(self.tempdir, 'xsync') self.processed_changelogs_dir = self.tempdir + self.name = "xsync" logging.info('xsync temp directory: %s' % self.tempdir) try: os.makedirs(self.tempdir) diff --git a/geo-replication/syncdaemon/monitor.py b/geo-replication/syncdaemon/monitor.py index ba5c8e32514..c41eb969143 100644 --- a/geo-replication/syncdaemon/monitor.py +++ b/geo-replication/syncdaemon/monitor.py @@ -99,6 +99,10 @@ class Volinfo(object): def get(self, elem): return self.tree.findall('.//' + elem) + def is_cold(self, brickpath): + logging.debug('brickpath: ' + repr(brickpath)) + return brickpath in self.cold_bricks + @property @memoize def bricks(self): @@ -127,6 +131,10 @@ class Volinfo(object): def disperse_count(self): return int(self.get('disperseCount')[0].text) + @property + @memoize + def cold_bricks(self): + return [b.text for b in self.get('coldBricks/brick')] class Monitor(object): @@ -248,9 +256,9 @@ class Monitor(object): '--rpc-fd', ','.join([str(rw), str(ww), str(ra), str(wa)]), - '--subvol-num', str(w[2]), - '--resource-remote', - remote_host]) + '--subvol-num', str(w[2])] + + (['--is-coldtier'] if w[3] else []) + + ['--resource-remote', remote_host]) cpids.add(cpid) agents.add(apid) @@ -377,7 +385,8 @@ def distribute(*resources): slaves = slavevols workerspex = [(brick['dir'], slaves[idx % len(slaves)], - get_subvol_num(idx, mvol.replica_count, mvol.disperse_count)) + get_subvol_num(idx, mvol.replica_count, mvol.disperse_count), + mvol.is_cold(":".join([brick['host'], brick['dir']]))) for idx, brick in enumerate(mvol.bricks) if is_host_local(brick['host'])] logging.info('worker specs: ' + repr(workerspex)) diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py index c73347aaf17..51f88627a96 100644 --- a/geo-replication/syncdaemon/resource.py +++ b/geo-replication/syncdaemon/resource.py @@ -1435,7 +1435,12 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote): # Note: if config.change_detector is xsync then # it will not use changelog history api try: - g3.crawlwrap(oneshot=True) + # if cold brick type, avoid changeloghistory and use xsync + if (boolify(gconf.is_coldtier)): + logging.info("cold tier using xsync crawl") + g1.crawlwrap(oneshot=True) + else: + g3.crawlwrap(oneshot=True) except PartialHistoryAvailable as e: logging.info('Partial history available, using xsync crawl' ' after consuming history till %s' % str(e)) |