syncdaemon: implement template substitutions for config values

So, for example, a log file setting of

  /var/log/${mastervol}/${eSlave}.log

will be substituted with the volume name of the master and the
canonicalized-escaped name of the slave for each master-slave pair.

As template expanders, beyond the various forms and derivatives
of master and slave, the following are also available:
 - gsyncd tunables (set in command line or in config)
 - for regexp sections, regexp group captures can be accessed
   via "match<i>_<n>", where i=1,2 corresponds to the i-th peer-rx
   in the section title and n=1,... to the n-th capture.

This will enable us to have a static configuration (not having to
add new entries on each gsyncd start).

Signed-off-by: Csaba Henk <csaba@lowlife.hu>
Signed-off-by: Anand Avati <avati@gluster.com>

BUG: 2785 (gsyncd logs on slave side go to /dev/null)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2785

3 files changed, 66 insertions, 22 deletions

diff --git a/xlators/features/marker/utils/syncdaemon/configinterface.py b/xlators/features/marker/utils/syncdaemon/configinterface.py
index fce45a2bb07..a170b2236cb 100644
--- a/xlators/features/marker/utils/syncdaemon/configinterface.py
+++ b/xlators/features/marker/utils/syncdaemon/configinterface.py
@@ -4,6 +4,7 @@ except ImportError:
     # py 3
     import configparser as ConfigParser
 import re
+from string import Template
 
 from syncdutils import escape, unescape, norm, update_file
 
@@ -13,6 +14,22 @@ config_version = 2.0
 
 re_type = type(re.compile(''))
 
+
+class MultiDict(object):
+
+    def __init__(self, *dd):
+        self.dicts = dd
+
+    def __getitem__(self, key):
+        val = None
+        for d in self.dicts:
+            if d.get(key):
+                val = d[key]
+        if not val:
+            raise KeyError(key)
+        return val
+
+
 class GConffile(object):
 
     def _normconfig(self):
@@ -26,9 +43,10 @@ class GConffile(object):
                 s2[k] = v
             self.config._sections[n] = s2
 
-    def __init__(self, path, peers):
+    def __init__(self, path, peers, *dd):
         self.peers = peers
         self.path = path
+        self.auxdicts = dd
         self.config = ConfigParser.RawConfigParser()
         self.config.read(path)
         self._normconfig()
@@ -89,27 +107,37 @@ class GConffile(object):
         ss.sort(scmp)
         return ss
 
-    def update_to(self, dct):
+    def update_to(self, dct, allow_unresolved=False):
         if not self.peers:
             raise RuntimeError('no peers given, cannot select matching options')
-        def update_from_sect(sect):
-            dct.update(self.config._sections[sect])
+        def update_from_sect(sect, mud):
+            for k, v in self.config._sections[sect].items():
+                if k == '__name__':
+                    continue
+                if allow_unresolved:
+                    dct[k] = Template(v).safe_substitute(mud)
+                else:
+                    dct[k] = Template(v).substitute(mud)
         for sect in self.ord_sections():
             sp = self.parse_section(sect)
             if isinstance(sp[0], re_type) and len(sp) == len(self.peers):
                 match = True
+                mad = {}
                 for i in range(len(sp)):
-                    if not sp[i].search(self.peers[i]):
+                    m = sp[i].search(self.peers[i])
+                    if not m:
                         match = False
                         break
+                    for j in range(len(m.groups())):
+                        mad['match%d_%d' % (i+1, j+1)] = m.groups()[j]
                 if match:
-                    update_from_sect(sect)
+                    update_from_sect(sect, MultiDict(dct, mad, *self.auxdicts))
         if self.config.has_section(self.section()):
-            update_from_sect(self.section())
+            update_from_sect(self.section(), MultiDict(dct, mad, *self.auxdicts))
 
     def get(self, opt=None):
         d = {}
-        self.update_to(d)
+        self.update_to(d, allow_unresolved = True)
         if opt:
             opt = norm(opt)
             v = d.get(opt)
diff --git a/xlators/features/marker/utils/syncdaemon/gsyncd.py b/xlators/features/marker/utils/syncdaemon/gsyncd.py
index 17c4508086e..76358414f89 100644
--- a/xlators/features/marker/utils/syncdaemon/gsyncd.py
+++ b/xlators/features/marker/utils/syncdaemon/gsyncd.py
@@ -183,6 +183,7 @@ def main_i():
     if getattr(confdata, 'rx', None):
         # peers are regexen, don't try to parse them
         canon_peers = args
+        namedict = {}
     else:
         rscs = [resource.parse_url(u) for u in args]
         dc = rconf.get('do_canon')
@@ -192,21 +193,35 @@ def main_i():
             return
         local = remote = None
         if rscs:
-          local = rscs[0]
-          if len(rscs) > 1:
-              remote = rscs[1]
-          if not local.can_connect_to(remote):
-              raise RuntimeError("%s cannot work with %s" % (local.path, remote and remote.path))
-        pa = ([], [])
-        canon = [False, True]
-        for x in (local, remote):
-            if x:
-                for i in range(2):
-                    pa[i].append(x.get_url(canonical=canon[i]))
-        peers, canon_peers = pa
+            local = rscs[0]
+            if len(rscs) > 1:
+                remote = rscs[1]
+            if not local.can_connect_to(remote):
+                raise RuntimeError("%s cannot work with %s" % (local.path, remote and remote.path))
+        pa = ([], [], [])
+        urlprms = ({}, {'canonical': True}, {'canonical': True, 'escaped': True})
+        for x in rscs:
+            for i in range(len(pa)):
+                pa[i].append(x.get_url(**urlprms[i]))
+        peers, canon_peers, canon_esc_peers = pa
+        # creating the namedict, a dict representing various ways of referring to / repreenting
+        # peers to be fillable in config templates
+        mods = (lambda x: x, lambda x: x[0].upper() + x[1:], lambda x: 'e' + x[0].upper() + x[1:])
+        if remote:
+            rmap = { local: ('local', 'master'), remote: ('remote', 'slave') }
+        else:
+            rmap = { local: ('local', 'slave') }
+        namedict = {}
+        for i in range(len(rscs)):
+            x = rscs[i]
+            for name in rmap[x]:
+                for j in range(3):
+                    namedict[mods[j](name)] = pa[j][i]
+                if x.scheme == 'gluster':
+                    namedict[name + 'vol'] = x.volume
     if not 'config_file' in rconf:
         rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]), "conf/gsyncd.conf")
-    gcnf = GConffile(rconf['config_file'], canon_peers)
+    gcnf = GConffile(rconf['config_file'], canon_peers, defaults.__dict__, opts.__dict__, namedict)
 
     if confdata:
         opt_ok = norm(confdata.opt) in tunables + [None]
diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/xlators/features/marker/utils/syncdaemon/resource.py
index 777ff599999..7db7d5bb40f 100644
--- a/xlators/features/marker/utils/syncdaemon/resource.py
+++ b/xlators/features/marker/utils/syncdaemon/resource.py
@@ -255,6 +255,7 @@ class AbstractUrl(object):
         self.path = path
         return m.groups()
 
+    @property
     def scheme(self):
         return type(self).__name__.lower()
 
@@ -266,7 +267,7 @@ class AbstractUrl(object):
             pa = self.canonical_path()
         else:
             pa = self.path
-        u = "://".join((self.scheme(), pa))
+        u = "://".join((self.scheme, pa))
         if escaped:
             u = syncdutils.escape(u)
         return u