georep: python2 to python3 compat - syscalls

1. ctypes/syscalls
    A) arguments is expected to be encoded
    B) Raw conversion of return value from bytearray into string
2. struct pack/unpack - Raw converstion of string to bytearray
3. basestring -> str

Updates: #411
Change-Id: I80f939adcdec0ed0022c87c0b76d057ad5559e5a
Signed-off-by: Kotresh HR <khiremat@redhat.com>

7 files changed, 88 insertions, 35 deletions

diff --git a/geo-replication/syncdaemon/Makefile.am b/geo-replication/syncdaemon/Makefile.am
index 19f0bfce1b7..62c5ce7fe30 100644
--- a/geo-replication/syncdaemon/Makefile.am
+++ b/geo-replication/syncdaemon/Makefile.am
@@ -3,6 +3,6 @@ syncdaemondir = $(GLUSTERFS_LIBEXECDIR)/python/syncdaemon
 syncdaemon_PYTHON = rconf.py gsyncd.py __init__.py master.py README.md repce.py \
 	resource.py syncdutils.py monitor.py libcxattr.py gsyncdconfig.py \
 	libgfchangelog.py changelogagent.py gsyncdstatus.py conf.py logutils.py \
-	subcmds.py argsupgrade.py
+	subcmds.py argsupgrade.py py2py3.py
 
 CLEANFILES =
diff --git a/geo-replication/syncdaemon/libcxattr.py b/geo-replication/syncdaemon/libcxattr.py
index 2d186b95c89..7f3f6ce453a 100644
--- a/geo-replication/syncdaemon/libcxattr.py
+++ b/geo-replication/syncdaemon/libcxattr.py
@@ -10,6 +10,8 @@
 
 import os
 from ctypes import CDLL, create_string_buffer, get_errno
+import py2py3
+from py2py3 import bytearray_to_str
 
 
 class Xattr(object):
@@ -38,20 +40,23 @@ class Xattr(object):
     @classmethod
     def _query_xattr(cls, path, siz, syscall, *a):
         if siz:
-            buf = create_string_buffer('\0' * siz)
+            buf = create_string_buffer(b'\0' * siz)
         else:
             buf = None
         ret = getattr(cls.libc, syscall)(*((path,) + a + (buf, siz)))
         if ret == -1:
             cls.raise_oserr()
         if siz:
-            return buf.raw[:ret]
+            # py2 and py3 compatibility. Convert bytes array
+            # to string
+            result = bytearray_to_str(buf.raw)
+            return result[:ret]
         else:
             return ret
 
     @classmethod
     def lgetxattr(cls, path, attr, siz=0):
-        return cls._query_xattr(path, siz, 'lgetxattr', attr)
+        return cls._query_xattr(path.encode(), siz, 'lgetxattr', attr.encode())
 
     @classmethod
     def lgetxattr_buf(cls, path, attr):
@@ -65,7 +70,7 @@ class Xattr(object):
 
     @classmethod
     def llistxattr(cls, path, siz=0):
-        ret = cls._query_xattr(path, siz, 'llistxattr')
+        ret = cls._query_xattr(path.encode(), siz, 'llistxattr')
         if isinstance(ret, str):
             ret = ret.strip('\0')
             ret = ret.split('\0') if ret else []
@@ -73,13 +78,13 @@ class Xattr(object):
 
     @classmethod
     def lsetxattr(cls, path, attr, val):
-        ret = cls.libc.lsetxattr(path, attr, val, len(val), 0)
+        ret = cls.libc.lsetxattr(path.encode(), attr.encode(), val, len(val), 0)
         if ret == -1:
             cls.raise_oserr()
 
     @classmethod
     def lremovexattr(cls, path, attr):
-        ret = cls.libc.lremovexattr(path, attr)
+        ret = cls.libc.lremovexattr(path.encode(), attr.encode())
         if ret == -1:
             cls.raise_oserr()
 
diff --git a/geo-replication/syncdaemon/libgfchangelog.py b/geo-replication/syncdaemon/libgfchangelog.py
index da12438d069..cc40fd5475d 100644
--- a/geo-replication/syncdaemon/libgfchangelog.py
+++ b/geo-replication/syncdaemon/libgfchangelog.py
@@ -39,8 +39,8 @@ class Changes(object):
 
     @classmethod
     def cl_register(cls, brick, path, log_file, log_level, retries=0):
-        ret = cls._get_api('gf_changelog_register')(brick, path,
-                                                    log_file,
+        ret = cls._get_api('gf_changelog_register')(brick.encode(), path.encode(),
+                                                    log_file.encode(),
                                                     log_level, retries)
         if ret == -1:
             cls.raise_changelog_err()
@@ -63,14 +63,14 @@ class Changes(object):
         def clsort(f):
             return f.split('.')[-1]
         changes = []
-        buf = create_string_buffer('\0', 4096)
+        buf = create_string_buffer(b'\0' * 4096)
         call = cls._get_api('gf_changelog_next_change')
 
         while True:
             ret = call(buf, 4096)
             if ret in (0, -1):
                 break
-            changes.append(buf.raw[:ret - 1])
+            changes.append(buf.raw[:ret - 1].decode())
         if ret == -1:
             cls.raise_changelog_err()
         # cleanup tracker
@@ -79,7 +79,7 @@ class Changes(object):
 
     @classmethod
     def cl_done(cls, clfile):
-        ret = cls._get_api('gf_changelog_done')(clfile)
+        ret = cls._get_api('gf_changelog_done')(clfile.encode())
         if ret == -1:
             cls.raise_changelog_err()
 
@@ -94,7 +94,7 @@ class Changes(object):
     @classmethod
     def cl_history_changelog(cls, changelog_path, start, end, num_parallel):
         actual_end = c_ulong()
-        ret = cls._get_api('gf_history_changelog')(changelog_path, start, end,
+        ret = cls._get_api('gf_history_changelog')(changelog_path.encode(), start, end,
                                                    num_parallel,
                                                    byref(actual_end))
         if ret == -1:
@@ -118,14 +118,14 @@ class Changes(object):
             return f.split('.')[-1]
 
         changes = []
-        buf = create_string_buffer('\0', 4096)
+        buf = create_string_buffer(b'\0' * 4096)
         call = cls._get_api('gf_history_changelog_next_change')
 
         while True:
             ret = call(buf, 4096)
             if ret in (0, -1):
                 break
-            changes.append(buf.raw[:ret - 1])
+            changes.append(buf.raw[:ret - 1].decode())
         if ret == -1:
             cls.raise_changelog_err()
 
@@ -133,6 +133,6 @@ class Changes(object):
 
     @classmethod
     def cl_history_done(cls, clfile):
-        ret = cls._get_api('gf_history_changelog_done')(clfile)
+        ret = cls._get_api('gf_history_changelog_done')(clfile.encode())
         if ret == -1:
             cls.raise_changelog_err()
diff --git a/geo-replication/syncdaemon/monitor.py b/geo-replication/syncdaemon/monitor.py
index ca2839059a3..c45ef24e59f 100644
--- a/geo-replication/syncdaemon/monitor.py
+++ b/geo-replication/syncdaemon/monitor.py
@@ -26,7 +26,9 @@ from syncdutils import set_term_handler, GsyncdError
 from syncdutils import Thread, finalize, Volinfo, VolinfoFromGconf
 from syncdutils import gf_event, EVENT_GEOREP_FAULTY, get_up_nodes
 from gsyncdstatus import GeorepStatus, set_monitor_status
-from syncdutils import unshare_propagation_supported, pipe
+from syncdutils import unshare_propagation_supported
+import py2py3
+from py2py3 import pipe
 
 ParseError = XET.ParseError if hasattr(XET, 'ParseError') else SyntaxError
 
diff --git a/geo-replication/syncdaemon/py2py3.py b/geo-replication/syncdaemon/py2py3.py
new file mode 100644
index 00000000000..4c0e1152aa8
--- /dev/null
+++ b/geo-replication/syncdaemon/py2py3.py
@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+# All python2/python3 compatibility routines
+
+import sys
+import os
+
+
+if sys.version_info >= (3,):
+    def pipe():
+        (r, w) = os.pipe()
+        os.set_inheritable(r, True)
+        os.set_inheritable(w, True)
+        return (r, w)
+
+    # Raw conversion of bytearray to string. Used in the cases where
+    # buffer is created by create_string_buffer which is a 8-bit char
+    # array and passed to syscalls to fetch results. Using encode/decode
+    # doesn't work as it converts to string altering the size.
+    def bytearray_to_str(byte_arr):
+        return ''.join([chr(b) for b in byte_arr])
+
+    # Raw conversion of string to bytes. This is required to convert
+    # back the string into bytearray(c char array) to use in struc
+    # pack/unpacking. Again encode/decode can't be used as it
+    # converts it alters size.
+    def str_to_bytearray(string):
+        return bytes([ord(c) for c in string])
+
+else:
+    def pipe():
+        (r, w) = os.pipe()
+        return (r, w)
+
+    # Raw conversion of bytearray to string
+    def bytearray_to_str(byte_arr):
+        return ''.join([b for b in byte_arr])
+
+    # Raw conversion of string to bytearray
+    def str_to_bytearray(string):
+        return b"".join([c for c in string])
diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py
index a0edeb7d56b..200b5566e08 100644
--- a/geo-replication/syncdaemon/resource.py
+++ b/geo-replication/syncdaemon/resource.py
@@ -40,7 +40,9 @@ from syncdutils import GX_GFID_CANONICAL_LEN
 from gsyncdstatus import GeorepStatus
 from syncdutils import lf, Popen, sup
 from syncdutils import Xattr, matching_disk_gfid, get_gfid_from_mnt
-from syncdutils import unshare_propagation_supported, get_slv_dir_path, pipe
+from syncdutils import unshare_propagation_supported, get_slv_dir_path
+import py2py3
+from py2py3 import pipe, str_to_bytearray
 
 
 ENOTSUP = getattr(errno, 'ENOTSUP', 'EOPNOTSUPP')
@@ -145,6 +147,7 @@ class Server(object):
         if buf == ENOENT:
             return buf
         else:
+            buf = str_to_bytearray(buf)
             m = re.match('(.{8})(.{4})(.{4})(.{4})(.{12})', "".join(
                 ['%02x' % x for x in struct.unpack(cls.GFID_FMTSTR, buf)]))
             return '-'.join(m.groups())
@@ -235,6 +238,7 @@ class Server(object):
             val = Xattr.lgetxattr(path,
                                   '.'.join([cls.GX_NSPACE, uuid, 'xtime']),
                                   8)
+            val = str_to_bytearray(val)
             return struct.unpack('!II', val)
         except OSError:
             ex = sys.exc_info()[1]
@@ -257,6 +261,7 @@ class Server(object):
             val = Xattr.lgetxattr(path,
                                   '.'.join([cls.GX_NSPACE, uuid, 'stime']),
                                   8)
+            val = str_to_bytearray(val)
             return struct.unpack('!II', val)
         except OSError:
             ex = sys.exc_info()[1]
@@ -279,6 +284,7 @@ class Server(object):
             val = Xattr.lgetxattr(path,
                                   '.'.join([cls.GX_NSPACE, uuid, 'stime']),
                                   8)
+            val = str_to_bytearray(val)
             return struct.unpack('!II', val)
         except OSError:
             ex = sys.exc_info()[1]
@@ -302,6 +308,7 @@ class Server(object):
                                   '.'.join([cls.GX_NSPACE, uuid,
                                             'entry_stime']),
                                   8)
+            val = str_to_bytearray(val)
             return struct.unpack('!II', val)
         except OSError:
             ex = sys.exc_info()[1]
@@ -375,7 +382,7 @@ class Server(object):
         def entry_pack_reg(gf, bn, mo, uid, gid):
             blen = len(bn)
             return struct.pack(cls._fmt_mknod(blen),
-                               uid, gid, gf, mo, bn,
+                               uid, gid, gf.encode(), mo, bn.encode(),
                                stat.S_IMODE(mo), 0, umask())
 
         def entry_pack_reg_stat(gf, bn, st):
@@ -383,14 +390,14 @@ class Server(object):
             mo = st['mode']
             return struct.pack(cls._fmt_mknod(blen),
                                st['uid'], st['gid'],
-                               gf, mo, bn,
+                               gf.encode(), mo, bn.encode(),
                                stat.S_IMODE(mo), 0, umask())
         # mkdir
 
         def entry_pack_mkdir(gf, bn, mo, uid, gid):
             blen = len(bn)
             return struct.pack(cls._fmt_mkdir(blen),
-                               uid, gid, gf, mo, bn,
+                               uid, gid, gf.encode(), mo, bn.encode(),
                                stat.S_IMODE(mo), umask())
         # symlink
 
@@ -399,7 +406,8 @@ class Server(object):
             llen = len(lnk)
             return struct.pack(cls._fmt_symlink(blen, llen),
                                st['uid'], st['gid'],
-                               gf, st['mode'], bn, lnk)
+                               gf.encode(), st['mode'], bn.encode(),
+                               lnk.encode())
 
         def entry_purge(op, entry, gfid, e):
             # This is an extremely racy code and needs to be fixed ASAP.
@@ -450,7 +458,7 @@ class Server(object):
                 else:
                     en = e['entry']
                 disk_gfid = get_gfid_from_mnt(en)
-                if isinstance(disk_gfid, basestring) and \
+                if isinstance(disk_gfid, str) and \
                    e['gfid'] != disk_gfid:
                     slv_entry_info['gfid_mismatch'] = True
                     st = lstat(en)
@@ -1022,6 +1030,7 @@ class GLUSTERServer(Server):
         """generic volume mark fetching/parsing backed"""
         fmt_string = cls.NTV_FMTSTR + extra_fields
         buf = Xattr.lgetxattr('.', xattr, struct.calcsize(fmt_string))
+        buf = str_to_bytearray(buf)
         vm = struct.unpack(fmt_string, buf)
         m = re.match(
             '(.{8})(.{4})(.{4})(.{4})(.{12})',
diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py
index cbb169c6c96..fad1a3e4f76 100644
--- a/geo-replication/syncdaemon/syncdutils.py
+++ b/geo-replication/syncdaemon/syncdutils.py
@@ -1021,15 +1021,3 @@ def get_up_nodes(hosts, port):
             up_nodes.append(h)
 
     return up_nodes
-
-
-def pipe():
-    # Pipe routine for python2 and python3 compatiability
-    try:
-        (r, w) = os.pipe()
-        os.set_inheritable(r, True)
-        os.set_inheritable(w, True)
-    except AttributeError:
-        (r, w) = os.pipe()
-
-    return (r, w)