Geo-rep: Adding regression tests for geo-rep

This patch introduces upstream regression suit for geo-replication

    * Modifies cleanup (tests/include.rc) to remove everything but
      hook-scripts.

Prerequisites:
    * Passwordless SSH from root to root of current host.
    * Export /build/install/sbin and /build/install/bin to PATH
      variable for root user.

Change-Id: I433dd8bbb17edba9baaf516fe0dce3133ba39184
BUG: 1101111
Signed-off-by: Vijaykumar Koppad <vkoppad@redhat.com>
Signed-off-by: Ajeet Jha <ajha@redhat.com>
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: http://review.gluster.org/7392
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Aravinda VK <avishwan@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

13 files changed, 1511 insertions, 129 deletions

diff --git a/tests/README.md b/tests/README.md
index 50e1a9d5005..09c98576987 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -5,6 +5,11 @@ Regression tests framework for GlusterFS
 - Build and install the version of glusterfs with your changes. Make
   sure the installed version is accessible from $PATH.
 
+## Prereq for geo-rep regression tests.
+- Passwordless ssh on the test system to itself
+- arequal-checksum installed on the test-system.
+  You can find the repo here - https://github.com/raghavendrabhat/arequal
+
 ## How-To
 - To mount glusterfs, NEVER use 'mount -t glusterfs', instead use
   'glusterfs -s ' method. This is because with the patch build setup
@@ -30,6 +35,7 @@ Regression tests framework for GlusterFS
       `tests/bugs/` directory.
     - a glob pattern (see `man 7 glob` for mor info on globs)
 
+- To execute single ".t" file, use "prove -vf /path/to/.t"
 - If some test cases fail, report to GlusterFS community at
   `gluster-devel@gluster.org`.
 
diff --git a/tests/geo-rep/compare-arequal.py b/tests/geo-rep/compare-arequal.py
new file mode 100644
index 00000000000..6d9850337b8
--- /dev/null
+++ b/tests/geo-rep/compare-arequal.py
@@ -0,0 +1,244 @@
+#!/usr/bin/python
+
+import sys
+import os
+import re
+import tempfile
+import subprocess
+from multiprocessing import Pool
+import time
+from optparse import OptionParser
+
+slave_dict = {}
+master_res = ''
+
+
+def get_arequal_checksum(me, mnt):
+    global slave_dict
+    master_cmd = ['./tests/utils/arequal-checksum', '-p', mnt]
+    print "Calculating  "+me+" checksum ..."
+    print ""
+    p = subprocess.Popen(master_cmd, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    ret = p.wait()
+    stdout, stderr = p.communicate()
+    if ret:
+        print "Failed to get the checksum of " + me + " with following error"
+        print stderr
+        return 1
+    else:
+        return stdout
+
+
+def get_file_count(me, mnt):
+    global slave_dict
+    master_cmd = ['find ' + mnt + ' |wc -l']
+    print "Calculating  " + me + " files ..."
+    print ""
+    p = subprocess.Popen(master_cmd, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE, shell=True)
+    ret = p.wait()
+    stdout, stderr = p.communicate()
+    if ret:
+        print "Failed to get the count of files in " + me
+        + " with following error"
+        print stderr
+        return 1
+    else:
+        return stdout.strip()
+
+
+def compare_checksum(master_mnt, slave_dict):
+    proc = len(slave_dict)+1
+    pool = Pool(processes=proc)
+    master_res = pool.apply_async(get_arequal_checksum, args=("master",
+                                                              master_mnt))
+    results = [(slave, pool.apply_async(get_arequal_checksum,
+                                        args=(slave_dict[slave]["vol"],
+                                              slave_dict[slave]["mnt"])))
+               for slave in slave_dict]
+
+    pool.close()
+    pool.join()
+    for slave, result in results:
+        slave_dict[slave]["res"] = result.get()
+        # exception:  OSError
+
+    master_res = master_res.get()
+
+    print "arequal-checksum of master is : \n %s" % master_res
+    for slave in slave_dict:
+        print "arequal-checksum of geo_rep_slave %s: \n %s" % (
+            slave_dict[slave]["vol"], slave_dict[slave]["res"])
+
+    master_files, master_total = re.findall('Total[\s]+:\s(\w+)', master_res)
+    master_reg_meta, master_reg = re.findall('Regular files[\s]+:\s(\w+)',
+                                             master_res)[1:]
+    master_dir_meta, master_dir = re.findall('Directories[\s]+:\s(\w+)',
+                                             master_res)[1:]
+
+    ret = 0
+    for slave in slave_dict:
+        slave_dict[slave]["files"], slave_dict[slave]["total"] = re.findall(
+            'Total[\s]+:\s(\w+)', slave_dict[slave]["res"])
+        slave_dict[slave]["reg_meta"], slave_dict[slave]["reg"] = re.findall(
+            'Regular files[\s]+:\s(\w+)', slave_dict[slave]["res"])[1:]
+        slave_dict[slave]["dir_meta"], slave_dict[slave]["dir"] = re.findall(
+            'Directories[\s]+:\s(\w+)', slave_dict[slave]["res"])[1:]
+
+        if master_reg_meta != slave_dict[slave]["reg_meta"]:
+            print ("Meta data checksum for regular files doesn't match " +
+                   "between master and  "+slave_dict[slave]["vol"])
+            ret = 67
+
+        if master_dir_meta != slave_dict[slave]["dir_meta"]:
+            print ("Meta data checksum for directories doesn't match " +
+                   "between master and "+slave_dict[slave]["vol"])
+            ret = 68
+
+        if master_files != slave_dict[slave]["files"]:
+            print ("Failed to sync all the files from master to " +
+                   slave_dict[slave]["vol"])
+            ret = 1
+
+        if master_total != slave_dict[slave]["total"]:
+            if master_reg != slave_dict[slave]["reg"]:
+                print ("Checksum for regular files doesn't match " +
+                       "between master and "+slave_dict[slave]["vol"])
+                ret = 1
+            elif master_dir != slave_dict[slave]["dir"]:
+                print ("Checksum for directories doesn't match between " +
+                       "master and "+slave_dict[slave]["vol"])
+                ret = 1
+            else:
+                print ("Checksum for symlinks or others doesn't match " +
+                       "between master and "+slave_dict[slave]["vol"])
+                ret = 1
+
+        if ret is 0:
+            print ("Successfully synced all the files from master " +
+                   "to the "+slave_dict[slave]["vol"])
+
+    return ret
+
+
+def compare_filecount(master_mnt, slave_dict):
+    proc = len(slave_dict)+1
+    pool = Pool(processes=proc)
+
+    master_res = pool.apply_async(get_file_count, args=("master", master_mnt))
+    results = [(slave, pool.apply_async(get_file_count,
+                                        args=(slave_dict[slave]["vol"],
+                                              slave_dict[slave]["mnt"])))
+               for slave in slave_dict]
+
+    pool.close()
+    pool.join()
+    for slave, result in results:
+        slave_dict[slave]["res"] = result.get()
+
+    master_res = master_res.get()
+    ret = 0
+    for slave in slave_dict:
+        if not master_res == slave_dict[slave]["res"]:
+            print ("files count between master and " +
+                   slave_dict[slave]["vol"]+" doesn't match yet")
+            ret = 1
+
+    return ret
+
+
+def parse_url(url):
+    match = re.search(r'([\w - _ @ \.]+)::([\w - _ @ \.]+)', url)
+    if match:
+        node = match.group(1)
+        vol = match.group(2)
+    else:
+        print 'given url is not a valid.'
+        sys.exit(1)
+    return node, vol
+
+
+def cleanup(master_mnt, slave_dict):
+    try:
+        os.system("umount %s" % (master_mnt))
+    except:
+        print("Failed to unmount the master volume")
+
+    for slave in slave_dict:
+
+        try:
+            os.system("umount %s" % (slave_dict[slave]["mnt"]))
+            os.removedirs(slave_dict[slave]["mnt"])
+        except:
+            print("Failed to unmount the "+slave+" volume")
+
+    os.removedirs(master_mnt)
+
+
+def main():
+
+    slaves = args[1:]
+
+    masterurl = args[0]
+    master_node, mastervol = parse_url(masterurl)
+    master_mnt = tempfile.mkdtemp()
+
+    i = 1
+    for slave in slaves:
+        slave_dict["slave"+str(i)] = {}
+        slave_dict["slave"+str(i)]["node"], slave_dict[
+            "slave"+str(i)]["vol"] = parse_url(slave)
+        slave_dict["slave"+str(i)]["mnt"] = tempfile.mkdtemp()
+        i += 1
+
+    try:
+        print ("mounting the master volume on "+master_mnt)
+        os.system("glusterfs -s  %s --volfile-id %s %s" % (master_node,
+                                                           mastervol,
+                                                           master_mnt))
+        time.sleep(3)
+    except:
+        print("Failed to mount the master volume")
+
+    for slave in slave_dict:
+        print slave
+        print slave_dict[slave]
+        try:
+            print ("mounting the slave volume on "+slave_dict[slave]['mnt'])
+            os.system("glusterfs -s %s --volfile-id %s %s" % (
+                slave_dict[slave]["node"], slave_dict[slave]["vol"],
+                slave_dict[slave]["mnt"]))
+            time.sleep(3)
+        except:
+            print("Failed to mount the "+slave+" volume")
+
+    res = 0
+    if option.check == "arequal":
+        res = compare_checksum(master_mnt, slave_dict)
+    elif option.check == "find":
+        res = compare_filecount(master_mnt, slave_dict)
+    else:
+        print "wrong options given"
+
+    cleanup(master_mnt, slave_dict)
+
+    sys.exit(res)
+
+
+if __name__ == '__main__':
+
+    usage = "usage: %prog [option] <master-host>::<master-vol> \
+    <slave1-host>::<slave1-vol> . . ."
+    parser = OptionParser(usage=usage)
+    parser.add_option("-c", dest="check", action="store", type="string",
+                      default="arequal",
+                      help="size of the files to be used [default: %default]")
+    (option, args) = parser.parse_args()
+    if not args:
+        print "usage: <script> [option] <master-host>::<master-vol>\
+         <slave1-host>::<slave1-vol> . . ."
+        print ""
+        sys.exit(1)
+
+    main()
diff --git a/tests/geo-rep/compare-gfid.py b/tests/geo-rep/compare-gfid.py
new file mode 100644
index 00000000000..432c1d77f31
--- /dev/null
+++ b/tests/geo-rep/compare-gfid.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+
+# Most of this script was written by M S Vishwanath Bhat (vbhat@redhat.com)
+
+import re
+import os
+import sys
+import xattr
+import tempfile
+
+
+def parse_url(url):
+    match = re.search(r'([\w - _ @ \.]+)::([\w - _ @ \.]+)', url)
+    if match:
+        node = match.group(1)
+        vol = match.group(2)
+    else:
+        print 'given url is not a valid url.'
+        sys.exit(1)
+    return node, vol
+
+
+def cleanup(master_mnt, slave_mnt):
+    try:
+        os.system("umount %s" % (master_mnt))
+    except:
+        print("Failed to unmount the master volume")
+    try:
+        os.system("umount %s" % (slave_mnt))
+    except:
+        print("Failed to unmount the slave volume")
+
+    os.removedirs(master_mnt)
+    os.removedirs(slave_mnt)
+
+
+def main():
+
+    masterurl = sys.argv[1]
+    slaveurl = sys.argv[2]
+    slave_node, slavevol = parse_url(slaveurl)
+    master_node, mastervol = parse_url(masterurl)
+
+    master_mnt = tempfile.mkdtemp()
+    slave_mnt = tempfile.mkdtemp()
+
+    try:
+        print "Mounting master volume on a temp mnt_pnt"
+        os.system("glusterfs -s %s --volfile-id %s %s" % (master_node,
+                                                          mastervol,
+                                                          master_mnt))
+    except:
+        print("Failed to mount the master volume")
+        cleanup(master_mnt, slave_mnt)
+        sys.exit(1)
+
+    try:
+        print "Mounting slave voluem on a temp mnt_pnt"
+        os.system("glusterfs -s %s --volfile-id %s %s" % (slave_node, slavevol,
+                                                          slave_mnt))
+    except:
+        print("Failed to mount the master volume")
+        cleanup(master_mnt, slave_mnt)
+        sys.exit(1)
+
+    slave_file_list = [slave_mnt]
+    for top, dirs, files in os.walk(slave_mnt, topdown=False):
+        for subdir in dirs:
+            slave_file_list.append(os.path.join(top, subdir))
+        for file in files:
+            slave_file_list.append(os.path.join(top, file))
+
+    # chdir and then get the gfid, so that you don't need to replace
+    gfid_attr = 'glusterfs.gfid'
+    ret = 0
+    for sfile in slave_file_list:
+        mfile = sfile.replace(slave_mnt, master_mnt)
+        if xattr.getxattr(sfile, gfid_attr, True) != xattr.getxattr(
+                mfile, gfid_attr, True):
+            print ("gfid of file %s in slave is different from %s" +
+                   " in master" % (sfile, mfile))
+            ret = 1
+
+    cleanup(master_mnt, slave_mnt)
+
+    sys.exit(ret)
+
+
+if __name__ == '__main__':
+    if len(sys.argv[1:]) < 2:
+        print ("Please pass master volume name and slave url as arguments")
+        print ("USAGE : python <script> <master-host>::<master-vol> " +
+               "<slave-host>::<slave-vol>")
+        sys.exit(1)
+    main()
diff --git a/tests/geo-rep/geo-rep-config.rc b/tests/geo-rep/geo-rep-config.rc
new file mode 100644
index 00000000000..53489082574
--- /dev/null
+++ b/tests/geo-rep/geo-rep-config.rc
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#LOG_DIR="$DEFAULT_LOG_FILE_DIRECTORY/glusterfs/geo-rep-auto-logs"
+LOG_DIR="/usr/local/var/log/geo-rep-auto-logs"
+LOG_FILE="$LOG_DIR/geo-rep-auto.log.`date +%Y%m%d-%H%M%S`"
+FILE_TYPE="text" # it can text, sparse or tar
+nf="5" # number of files in each directory when DIR_STR is MULTI
+ns="500" # number of files when DIR_STR is SINGLE
+DIR_STR="MULTI" # It can be either SINGLE or MULTI
+
+# Not using this option for now, can be used in the future.
+SYNC_MODE="rsync" # this option can take another option "tarssh"
+
+mkdir -p $LOG_DIR
diff --git a/tests/geo-rep/geo-rep-helper.rc b/tests/geo-rep/geo-rep-helper.rc
new file mode 100644
index 00000000000..de8a6817305
--- /dev/null
+++ b/tests/geo-rep/geo-rep-helper.rc
@@ -0,0 +1,296 @@
+#!/bin/bash
+
+function geo_rep_checkpoint_status()
+{
+    echo "Verifying the sync status using geo-rep checkpoint" >> $LOG_FILE
+    local timeout=300
+    local temp_status="NOTOK"
+    echo  "setting the checkpoint" >> $LOG_FILE
+
+    $CLI volume geo-rep $GMV0 $H0::$GSV0 config checkpoint now >> $LOG_FILE 2>&1
+
+# There is a bug, where in after checkpoint set, geo-rep status still
+# shows the old data for the first execution of geo-rep status. Running
+#geo-rep status to clear that.
+    $CLI volume geo-replication  $GMV0 $H0::$GSV0 status >> $LOG_FILE 2>&1
+
+    while [ $timeout -gt 0 ] && [ $temp_status == "NOTOK" ];
+    do
+	$CLI volume geo-replication $GMV0 $H0::$GSV0 status | \
+	egrep -i "not reached yet" 2>&1 >/dev/null
+	test $? -ne 0 && temp_status="completed"
+	echo "Waiting for the files to sync ..." >> $LOG_FILE
+	sleep 20
+	timeout=`expr $timeout - 20`
+	echo "temp_status is $temp_status" >> $LOG_FILE
+	echo "geo-rep status output:" >> $LOG_FILE
+	$CLI volume geo-replication $GMV0 $H0::$GSV0 status detail >> \
+	    $LOG_FILE 2>&1
+
+    done
+
+    echo "resetting the geo-rep checkpoint" >> $LOG_FILE
+    $CLI volume geo-rep $GMV0 $H0::$GSV0 config \!checkpoint  >> $LOG_FILE 2>&1
+
+    if test $temp_status = "completed" ; then
+	echo "geo-rep checkpoint has completed" >> $LOG_FILE
+	return 0
+    elif test $temp_status = "NOTOK" ; then
+	echo "geo-rep checkpoint has failed to complete within 300 seconds" >> \
+	    $LOG_FILE
+	return 1
+    fi
+}
+
+
+
+function geo_rep_arequal_status()
+{
+
+    echo "Verifying the sync status using arequal" >> $LOG_FILE
+    local timeout=300
+    local temp_status="NOTOK"
+    local comp_arequal="$(dirname $0)/compare-arequal.py"
+
+    while [ $timeout -gt 0 ] && [ $temp_status == "NOTOK" ];
+    do
+	echo "Waiting for the files to sync ..." >> $LOG_FILE
+	sleep 20
+	timeout=`expr $timeout - 20`
+
+	echo  "calculating and comparing arequal checksum between $GMV0 and \
+$GSV0 " >> $LOG_FILE
+	python $comp_arequal $H0::$GMV0 $H0::$GSV0 >> $LOG_FILE 2>&1
+
+	local ret=$?
+	# There is a bug, where sometimes metadata checksum of directories
+	# and regular files don't match. This is to avoid that for now.
+	if [[ $ret -eq 0 || $ret -eq 67 || $ret -eq 68 ]] ;then
+	    temp_status="completed"
+	fi
+
+    done
+
+    if test $temp_status = "completed" ; then
+	echo "checksum between master and slave match " >> $LOG_FILE
+	return 0
+    elif test $temp_status = "NOTOK" ; then
+	echo "checksum between master and slave doesn't match" >> $LOG_FILE
+	return 1
+    fi
+}
+
+
+function geo_rep_filecount_status()
+{
+
+    echo "Verifying the sync status through files count" >> $LOG_FILE
+    local timeout=300
+    local temp_status="NOTOK"
+    local comp_arequal="$(dirname $0)/compare-arequal.py"
+
+    while [ $timeout -gt 0 ] && [ $temp_status == "NOTOK" ];
+    do
+	echo "Waiting for the files to sync ..." >> $LOG_FILE
+	sleep 20
+	timeout=`expr $timeout - 20`
+
+	echo  "calculating and comparing files count  between $GMV0 and \
+$GSV0 " >> $LOG_FILE
+	python $comp_arequal -c "find" $H0::$GMV0 $H0::$GSV0 >> $LOG_FILE 2>&1
+
+	if [ $? -eq 0 ];then
+	    temp_status="completed"
+	fi
+
+    done
+
+    if test $temp_status = "completed" ; then
+	echo "files count  between master and slave match " >> $LOG_FILE
+	return 0
+    elif test $temp_status = "NOTOK" ; then
+	echo "files count between master and slave doesn't match" >> $LOG_FILE
+	return 1
+    fi
+}
+
+
+
+function check_status_arequal()
+{
+# checkpoint is failing to reach even though all the files got synced in the latest build.
+# Hence not using checkpoint to check for sync status.
+#    geo_rep_checkpoint_status
+    local comp_arequal="$(dirname $0)/compare-arequal.py"
+    local comp_gfid="$(dirname $0)/compare-gfid.py"
+
+    geo_rep_filecount_status
+
+    geo_rep_arequal_status
+
+    echo  "calculating and comparing gfids between $GMV0 and $GSV0 " \
+	>> $LOG_FILE
+    python  $comp_gfid $H0::$GMV0 $H0::$GSV0 >> $LOG_FILE 2>&1
+
+    if [ $? != 0 ]; then
+	return 1
+    else
+	echo "gfids between master and slave match" >> $LOG_FILE
+    fi
+
+    echo  "calculating and comparing arequal checksum between $GMV0 and $GSV0 " \
+	>> $LOG_FILE
+    python $comp_arequal $H0::$GMV0 $H0::$GSV0 >> $LOG_FILE 2>&1
+
+    local rett=$?
+
+    if [[ $rett -eq 0 || $rett -eq 67 || $rett -eq 68 ]] ;then
+	reta=0
+    else
+	reta=1
+    fi
+
+    return $reta
+
+}
+
+
+
+
+function create_data()
+{
+    fop=$1
+    MNT_PNT=$2
+    create_data="$(dirname $0)/../utils/create-files.py"
+
+    if [ $DIR_STR == "MULTI" ];then
+
+	python $create_data -n $nf --multi -b 10 -d 10 --random --max=2K \
+	    --min=1K -t $FILE_TYPE --fop=$fop $MNT_PNT >> $LOG_FILE 2>&1
+
+    elif [ $DIR_STR == "SINGLE" ];then
+
+	python $create_data -n $ns --random --max=2K --min=1K -t $FILE_TYPE \
+	    --fop=$fop $MNT_PNT >> $LOG_FILE 2>&1
+
+    else
+
+	echo "Wrong option for the create-files" >> $LOG_FILE
+
+    fi
+
+}
+
+
+function result()
+{
+
+local ret=$1
+local test=$2
+if [ $ret -ne 0 ]; then
+    echo -e "\n[ FAIL ] : $test has failed" >> $LOG_FILE
+    exit 1
+else
+    echo -e "\n[ PASS ] : $test has passed" >> $LOG_FILE
+fi
+
+}
+
+
+## hybrid crawl test-cases
+
+function hybrid_mode_test()
+{
+    local FOP=$1
+    local MNT_PNT=$2
+    echo -e "\n:::::::::::::::::::::::" >> $LOG_FILE
+    echo "Start of hybrid-mode-$DIR_STR-$FILE_TYPE-$FOP-test with \
+$MNT_PNT client" >> $LOG_FILE
+    echo -e ":::::::::::::::::::::::\n" >> $LOG_FILE
+
+    local ret=0
+    echo "stopping geo-rep session before creating data" >> $LOG_FILE
+
+    $CLI volume geo-rep $GMV0 $H0::$GSV0 stop force >> $LOG_FILE 2>&1
+
+    if [ $? -ne 0 ]; then
+	echo "stopping geo-rep session has failed" >> $LOG_FILE
+	return 1
+    fi
+
+    create_data $FOP $MNT_PNT
+
+    $CLI volume geo-rep $GMV0 $H0::$GSV0 start >> $LOG_FILE 2>&1
+
+    if [ $? -ne 0 ]; then
+	echo "starting geo-rep session has failed" >> $LOG_FILE
+	return 1
+    fi
+
+    check_status_arequal
+
+    if [ $? -ne 0 ]; then
+	ret=1
+    fi
+
+    result $ret "hybrid-mode-$DIR_STR-$FILE_TYPE-$FOP-test with $CLIENT client"
+
+    return $ret
+
+}
+
+#### Changelog based test-cases
+
+function changelog_mode_test()
+{
+    local FOP=$1
+    local MNT_PNT=$2
+    echo -e "\n:::::::::::::::::::::::" >> $LOG_FILE
+    echo "Start of changelog-mode-$DIR_STR-$FILE_TYPE-$FOP-test with \
+$MNT_PNT client" >> $LOG_FILE
+    echo -e ":::::::::::::::::::::::\n" >> $LOG_FILE
+
+    local ret=0
+
+    create_data $FOP $MNT_PNT
+
+    check_status_arequal
+
+    if [ $? -ne 0 ]; then
+	ret=1
+    fi
+
+    result $ret "basic-changelog-$DIR_STR-$FILE_TYPE-$FOP-test with $CLIENT \
+client"
+
+    return $ret
+}
+
+
+function changelog_mode_remove_test()
+{
+    MNT_PNT=$1
+
+    echo -e "\n:::::::::::::::::::::::" >> $LOG_FILE
+    echo "Start of changelog-mode-$DIR_STR-$FILE_TYPE-remove-test with \
+$MNT_PNT client" >> $LOG_FILE
+    echo -e ":::::::::::::::::::::::\n" >> $LOG_FILE
+
+    local ret=0
+
+    if [ ! -z $MNT_PNT ]; then
+	rm -rvf $MNT_PNT >> $LOG_FILE
+    else
+	echo "Value of MNT_PNT is NULL" >> $LOG_FILE
+    fi
+
+    check_status_arequal
+    if [ $? -ne 0 ]; then
+	ret=1
+    fi
+
+    result $ret "chnagelog-mode-$DIR_STR-$FILE_TYPE-remove-test with \
+$MNT_PNT client"
+
+    return $ret
+}
diff --git a/tests/geo-rep/georep-rsync-changelog.t b/tests/geo-rep/georep-rsync-changelog.t
new file mode 100644
index 00000000000..eda06a36df6
--- /dev/null
+++ b/tests/geo-rep/georep-rsync-changelog.t
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Following tests involves geo-rep regresseion tests with changelog
+# as change detector, and rsync as sync mode on both fuse and nfs mount
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/geo-rep-helper.rc
+. $(dirname $0)/geo-rep-config.rc
+
+cleanup;
+AREQUAL_PATH=$(dirname $0)/../utils
+CFLAGS=""
+test "`uname -s`" != "Linux" && {
+    CFLAGS="$CFLAGS -I$(dirname $0)/../../../contrib/argp-standalone ";
+    CFLAGS="$CFLAGS -L$(dirname $0)/../../../contrib/argp-standalone -largp ";
+    CFLAGS="$CFLAGS -lintl";
+}
+build_tester $AREQUAL_PATH/arequal-checksum.c $CFLAGS
+
+TEST glusterd
+TEST pidof glusterd
+
+setup_georep ;
+
+# start of tests on fuse mount
+
+TEST glusterfs -s $H0 --volfile-id $GMV0 $M0
+
+TEST changelog_mode_test "create" $M0
+
+TEST changelog_mode_test "chmod" $M0
+
+TEST changelog_mode_test "chown" $M0
+
+TEST changelog_mode_test "chgrp" $M0
+
+# Bug 1083963
+#TEST changelog_mode_test "rename" $M0
+
+TEST changelog_mode_test "truncate" $M0
+
+TEST changelog_mode_test "symlink" $M0
+
+# Bug 1003020
+#TEST changelog_mode_test "hardlink" $M0
+
+#TEST changelog_mode_remove_test $M0
+
+# start of tests on nfs mount
+
+TEST mount -t nfs -o vers=3,nolock $H0:$GMV0 $N0
+
+TEST changelog_mode_test "create" $N0
+
+TEST changelog_mode_test "chmod" $N0
+
+TEST changelog_mode_test "chown" $N0
+
+TEST changelog_mode_test "chgrp" $N0
+
+#TEST changelog_mode_test "rename" $N0
+
+TEST changelog_mode_test "truncate" $N0
+
+TEST changelog_mode_test "symlink" $N0
+
+#TEST changelog_mode_test "hardlink" $N0
+
+#TEST changelog_mode_remove_test $N0
+
+TEST rm -rf $AREQUAL_PATH/arequal-checksum
+cleanup_georep;
diff --git a/tests/geo-rep/georep-rsync-hybrid.t b/tests/geo-rep/georep-rsync-hybrid.t
new file mode 100644
index 00000000000..af23a526ffb
--- /dev/null
+++ b/tests/geo-rep/georep-rsync-hybrid.t
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Following tests involves geo-rep tests with hybrid crawl
+# as change detector, and rsync as sync mode on both fuse and nfs mount
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/geo-rep-helper.rc
+. $(dirname $0)/geo-rep-config.rc
+
+cleanup;
+
+AREQUAL_PATH=$(dirname $0)/../utils
+CFLAGS=""
+test "`uname -s`" != "Linux" && {
+    CFLAGS="$CFLAGS -I$(dirname $0)/../../../contrib/argp-standalone ";
+    CFLAGS="$CFLAGS -L$(dirname $0)/../../../contrib/argp-standalone -largp ";
+    CFLAGS="$CFLAGS -lintl";
+}
+build_tester $AREQUAL_PATH/arequal-checksum.c $CFLAGS
+
+TEST glusterd
+TEST pidof glusterd
+
+setup_georep ;
+
+# start of tests on fuse mount
+
+TEST glusterfs -s $H0 --volfile-id $GMV0 $M0
+
+TEST hybrid_mode_test "create" $M0
+
+TEST hybrid_mode_test "chmod" $M0
+
+TEST hybrid_mode_test "chown" $M0
+
+TEST hybrid_mode_test "chgrp" $M0
+
+TEST hybrid_mode_test "truncate" $M0
+
+TEST hybrid_mode_test "symlink" $M0
+
+#TEST hybrid_mode_test "hardlink" $M0
+
+# start of tests on nfs mount
+
+TEST mount -t nfs -o vers=3,nolock $H0:$GMV0 $N0
+
+TEST hybrid_mode_test "create" $N0
+
+TEST hybrid_mode_test "chmod" $N0
+
+TEST hybrid_mode_test "chown" $N0
+
+TEST hybrid_mode_test "chgrp" $N0
+
+TEST hybrid_mode_test "truncate" $N0
+
+TEST hybrid_mode_test "symlink" $N0
+
+#TEST hybrid_mode_test "hardlink" $N0
+
+TEST rm -rf $AREQUAL_PATH/arequal-checksum
+
+cleanup_georep;
diff --git a/tests/geo-rep/georep-setup.t b/tests/geo-rep/georep-setup.t
new file mode 100644
index 00000000000..75c379c8486
--- /dev/null
+++ b/tests/geo-rep/georep-setup.t
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $GMV0 replica 2  $H0:$B0/${GMV0}{1,2,3,4};
+
+TEST $CLI volume start $GMV0
+
+TEST $CLI volume create $GSV0 replica 2  $H0:$B0/${GSV0}{1,2,3,4};
+
+TEST $CLI volume start $GSV0
+
+TEST $CLI system:: execute gsec_create
+
+TEST $CLI volume geo-rep $GMV0 $H0::$GSV0 create push-pem
+
+TEST $CLI volume geo-rep $GMV0 $H0::$GSV0 start
+
+sleep 80 # after start geo-rep takes a minute to get stable
+
+TEST ! "$CLI volume geo-rep $GMV0 $H0::$GSV0 status | egrep -i 'faulty'"
+
+TEST  "$CLI volume geo-rep $GMV0 $H0::$GSV0 status | egrep -i 'Changelog crawl'"
+
+TEST $CLI volume geo-rep $GMV0 $H0::$GSV0 stop
+
+TEST $CLI volume geo-rep $GMV0 $H0::$GSV0 delete
diff --git a/tests/geo-rep/georep-tarssh-changelog.t b/tests/geo-rep/georep-tarssh-changelog.t
new file mode 100644
index 00000000000..1f0e817f551
--- /dev/null
+++ b/tests/geo-rep/georep-tarssh-changelog.t
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Following tests involves geo-rep regresseion tests with changelog
+# as change detector, and rsync as sync mode on both fuse and nfs mount
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/geo-rep-helper.rc
+. $(dirname $0)/geo-rep-config.rc
+
+cleanup;
+
+AREQUAL_PATH=$(dirname $0)/../utils
+CFLAGS=""
+test "`uname -s`" != "Linux" && {
+    CFLAGS="$CFLAGS -I$(dirname $0)/../../../contrib/argp-standalone ";
+    CFLAGS="$CFLAGS -L$(dirname $0)/../../../contrib/argp-standalone -largp ";
+    CFLAGS="$CFLAGS -lintl";
+}
+build_tester $AREQUAL_PATH/arequal-checksum.c $CFLAGS
+
+TEST glusterd
+TEST pidof glusterd
+
+setup_georep ;
+
+# start of tests on fuse mount
+
+TEST glusterfs -s $H0 --volfile-id $GMV0 $M0
+
+TEST changelog_mode_test "create" $M0
+
+TEST changelog_mode_test "chmod" $M0
+
+TEST changelog_mode_test "chown" $M0
+
+TEST changelog_mode_test "chgrp" $M0
+
+#TEST changelog_mode_test "rename" $M0
+
+TEST changelog_mode_test "truncate" $M0
+
+TEST changelog_mode_test "symlink" $M0
+
+#TEST changelog_mode_test "hardlink" $M0
+
+#TEST changelog_mode_remove_test $M0
+
+# start of tests on nfs mount
+
+TEST mount -t nfs -o vers=3,nolock $H0:$GMV0 $N0
+
+TEST changelog_mode_test "create" $N0
+
+TEST changelog_mode_test "chmod" $N0
+
+TEST changelog_mode_test "chown" $N0
+
+TEST changelog_mode_test "chgrp" $N0
+
+#TEST changelog_mode_test "rename" $N0
+
+TEST changelog_mode_test "truncate" $N0
+
+TEST changelog_mode_test "symlink" $N0
+
+#TEST changelog_mode_test "hardlink" $N0
+
+#TEST changelog_mode_remove_test $N0
+
+TEST rm -rf $AREQUAL_PATH/arequal-checksum
+
+cleanup_georep;
diff --git a/tests/geo-rep/georep-tarssh-hybrid.t b/tests/geo-rep/georep-tarssh-hybrid.t
new file mode 100644
index 00000000000..af23a526ffb
--- /dev/null
+++ b/tests/geo-rep/georep-tarssh-hybrid.t
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Following tests involves geo-rep tests with hybrid crawl
+# as change detector, and rsync as sync mode on both fuse and nfs mount
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/geo-rep-helper.rc
+. $(dirname $0)/geo-rep-config.rc
+
+cleanup;
+
+AREQUAL_PATH=$(dirname $0)/../utils
+CFLAGS=""
+test "`uname -s`" != "Linux" && {
+    CFLAGS="$CFLAGS -I$(dirname $0)/../../../contrib/argp-standalone ";
+    CFLAGS="$CFLAGS -L$(dirname $0)/../../../contrib/argp-standalone -largp ";
+    CFLAGS="$CFLAGS -lintl";
+}
+build_tester $AREQUAL_PATH/arequal-checksum.c $CFLAGS
+
+TEST glusterd
+TEST pidof glusterd
+
+setup_georep ;
+
+# start of tests on fuse mount
+
+TEST glusterfs -s $H0 --volfile-id $GMV0 $M0
+
+TEST hybrid_mode_test "create" $M0
+
+TEST hybrid_mode_test "chmod" $M0
+
+TEST hybrid_mode_test "chown" $M0
+
+TEST hybrid_mode_test "chgrp" $M0
+
+TEST hybrid_mode_test "truncate" $M0
+
+TEST hybrid_mode_test "symlink" $M0
+
+#TEST hybrid_mode_test "hardlink" $M0
+
+# start of tests on nfs mount
+
+TEST mount -t nfs -o vers=3,nolock $H0:$GMV0 $N0
+
+TEST hybrid_mode_test "create" $N0
+
+TEST hybrid_mode_test "chmod" $N0
+
+TEST hybrid_mode_test "chown" $N0
+
+TEST hybrid_mode_test "chgrp" $N0
+
+TEST hybrid_mode_test "truncate" $N0
+
+TEST hybrid_mode_test "symlink" $N0
+
+#TEST hybrid_mode_test "hardlink" $N0
+
+TEST rm -rf $AREQUAL_PATH/arequal-checksum
+
+cleanup_georep;
diff --git a/tests/include.rc b/tests/include.rc
index ed1fef0ae11..d4772bd42f4 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -5,6 +5,8 @@ N0=${N0:=/mnt/nfs/0};         # 0th mount point for NFS
 N1=${N1:=/mnt/nfs/1};         # 1st mount point for NFS
 V0=${V0:=patchy};             # volume name to use in tests
 V1=${V1:=patchy1};            # volume name to use in tests
+GMV0=${GMV0:=master};	      # master volume name to use in geo-rep tests
+GSV0=${GSV0:=slave};	      # slave volume name to use in geo-rep tests
 B0=${B0:=/d/backends};        # top level of brick directories
 WORKDIRS="$B0 $M0 $M1 $M2 $N0 $N1" 
 CC=cc
@@ -436,9 +438,11 @@ function cleanup()
                 ;;
         esac
 
-        if [ -n "${GLUSTERD_WORKDIR}" ] ; then
-            rm -rf $GLUSTERD_WORKDIR/* $B0/* /etc/glusterd/*;
-        fi
+        # remove contents of "GLUSTERD_WORKDIR" except hooks directory.
+        find  $GLUSTERD_WORKDIR/* -maxdepth 0 -name 'hooks' -prune \
+        -o -exec rm -rf '{}' ';'
+
+        rm -rf $B0/* /etc/glusterd/*;
 
         # unmount all stale mounts from /tmp, This is a temporary work around
         # till the stale mount in /tmp is found.
diff --git a/tests/utils/create-files.py b/tests/utils/create-files.py
index 05cf1279999..bef4201bf1f 100755
--- a/tests/utils/create-files.py
+++ b/tests/utils/create-files.py
@@ -5,107 +5,312 @@
 # http://github.com/vijaykumar-koppad/crefi
 
 from __future__ import with_statement
-import sys
 import os
 import re
-import random
-from optparse import OptionParser
+import sys
 import time
-import string
 import errno
+import xattr
+import string
+import random
+import logging
+import tarfile
+import argparse
+
+datsiz = 0
+timr = 0
+
+
+def setLogger(filename):
+    global logger
+    logger = logging.getLogger(filename)
+    logger.setLevel(logging.DEBUG)
+    return
+
+
+def setupLogger(filename):
+    logger = logging.getLogger(filename)
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(message)s')
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.INFO)
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+    return logger
+
 
 def os_rd(src, size):
-    fd = os.open(src,os.O_RDONLY)
+    global datsiz
+    fd = os.open(src, os.O_RDONLY)
     data = os.read(fd, size)
     os.close(fd)
+    datsiz = datsiz + size
     return data
 
+
 def os_wr(dest, data):
-    fd = os.open(dest,os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0644)
+    global timr
+    st = time.time()
+    fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0644)
     os.write(fd, data)
     os.close(fd)
+    ed = time.time()
+    timr = timr+(ed-st)
     return
 
-def create_sparse_file(fil):
-    if option.size:
-        option.random = False
-        size = option.size
+
+def create_sparse_file(fil, size, mins, maxs, rand):
+    if rand:
+        size = random.randint(mins, maxs)
     else:
-        size = random.randint(option.min, option.max)
+        size = size
     data = os_rd("/dev/zero", size)
     os_wr(fil, data)
     return
 
-def create_binary_file(fil):
-    if option.size:
-        option.random = False
-        size = option.size
+
+def create_binary_file(fil, size, mins, maxs, rand):
+    if rand:
+        size = random.randint(mins, maxs)
     else:
-        size = random.randint(option.min, option.max)
+        size = size
     data = os_rd("/dev/urandom", size)
     os_wr(fil, data)
     return
 
-def create_txt_file(fil):
-    if option.size:
-        option.random = False
-        size = option.size
-    else:
-        size = random.randint(option.min, option.max)
+
+def create_txt_file(fil, size, mins, maxs, rand):
+    if rand:
+        size = random.randint(mins, maxs)
     if size < 500*1024:
         data = os_rd("/etc/services", size)
         os_wr(fil, data)
     else:
-        data = os_rd("/etc/services", 500*1024)
+        data = os_rd("/etc/services", 512*1024)
         file_size = 0
-        fd = os.open(fil,os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0644)
+        fd = os.open(fil, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0644)
         while file_size < size:
             os.write(fd, data)
             file_size += 500*1024
         os.close(fd)
     return
 
-def get_filename():
-    size = option.flen
+
+def create_tar_file(fil, size, mins, maxs, rand):
+    if rand:
+        size = random.randint(mins, maxs)
+    else:
+        size = size
+    data = os_rd("/dev/urandom", size)
+    os_wr(fil, data)
+    tar = tarfile.open(fil+".tar.gz",  "w:gz")
+    tar.add(fil)
+    tar.close()
+    os.unlink(fil)
+    return
+
+
+def get_filename(flen):
+    size = flen
     char = string.uppercase+string.digits
     st = ''.join(random.choice(char) for i in range(size))
     ti = str((hex(int(str(time.time()).split('.')[0])))[2:])
-    return ti+"~~"+st
+    return ti+"%%"+st
+
+
+def text_files(files, file_count, inter, size, mins, maxs, rand,
+               flen, randname, dir_path):
+    global datsiz, timr
+    for k in range(files):
+        if not file_count % inter:
+            logger.info("Total files created -- "+str(file_count))
+        if not randname:
+            fil = dir_path+"/"+"file"+str(k)
+        else:
+            fil = dir_path+"/"+get_filename(flen)
+        create_txt_file(fil, size, mins, maxs, rand)
+        file_count += 1
+    return file_count
 
-def text_files(files, file_count):
+
+def sparse_files(files, file_count, inter, size, mins, maxs,
+                 rand, flen, randname, dir_path):
     for k in range(files):
-        if not file_count%option.inter:
-            print file_count
-        fil = get_filename()
-        create_txt_file(fil)
+        if not file_count % inter:
+            logger.info("Total files created -- "+str(file_count))
+        if not randname:
+            fil = dir_path+"/"+"file"+str(k)
+        else:
+            fil = dir_path+"/"+get_filename(flen)
+        create_sparse_file(fil, size, mins, maxs, rand)
         file_count += 1
     return file_count
 
-def sparse_files(files, file_count):
+
+def binary_files(files, file_count, inter, size, mins, maxs,
+                 rand, flen, randname, dir_path):
     for k in range(files):
-        if not file_count%option.inter:
-            print file_count
-        fil = get_filename()
-        create_sparse_file(fil)
+        if not file_count % inter:
+            logger.info("Total files created -- "+str(file_count))
+        if not randname:
+            fil = dir_path+"/"+"file"+str(k)
+        else:
+            fil = dir_path+"/"+get_filename(flen)
+        create_binary_file(fil, size, mins, maxs, rand)
         file_count += 1
     return file_count
 
-def binary_files(files, file_count):
+
+def tar_files(files, file_count, inter, size, mins, maxs,
+              rand, flen, randname, dir_path):
     for k in range(files):
-        if not file_count%option.inter:
-            print file_count
-        fil = get_filename()
-        create_binary_file(fil)
+        if not file_count % inter:
+            logger.info("Total files created -- "+str(file_count))
+        if not randname:
+            fil = dir_path+"/"+"file"+str(k)
+        else:
+            fil = dir_path+"/"+get_filename(flen)
+        create_tar_file(fil, size, mins, maxs, rand)
         file_count += 1
     return file_count
 
+
+def setxattr_files(files, randname, dir_path):
+    char = string.uppercase+string.digits
+    if not randname:
+        for k in range(files):
+            v = ''.join(random.choice(char) for i in range(10))
+            n = "user."+v
+            xattr.setxattr(dir_path+"/"+"file"+str(k), n, v)
+    else:
+        dirs = os.listdir(dir_path+"/")
+        for fil in dirs:
+            v = ''.join(random.choice(char) for i in range(10))
+            n = "user."+v
+            xattr.setxattr(dir_path+"/"+fil, n, v)
+    return
+
+
+def rename_files(files, flen, randname, dir_path):
+    if not randname:
+        for k in range(files):
+            os.rename(dir_path + "/" + "file" + str(k),
+                      dir_path + "/" + "file" + str(files+k))
+    else:
+        dirs = os.listdir(dir_path)
+        for fil in dirs:
+            if not os.path.isdir(fil):
+                newfil = get_filename(flen)
+                os.rename(dir_path + "/" + fil,
+                          dir_path + "/" + newfil)
+    return
+
+
+def truncate_files(files, mins, maxs, randname, dir_path):
+    if not randname:
+        for k in range(files):
+            byts = random.randint(mins, maxs)
+            fd = os.open(dir_path + "/" + "file" + str(k), os.O_WRONLY)
+            os.ftruncate(fd, byts)
+            os.close(fd)
+    else:
+        dirs = os.listdir(dir_path)
+        for fil in dirs:
+            if not os.path.isdir(dir_path+"/"+fil):
+                byts = random.randint(mins, maxs)
+                fd = os.open(dir_path+"/"+fil, os.O_WRONLY)
+                os.ftruncate(fd, byts)
+                os.close(fd)
+    return
+
+
+def chmod_files(files, flen, randname, dir_path):
+    if not randname:
+        for k in range(files):
+            mod = random.randint(0, 511)
+            os.chmod(dir_path+"/"+"file"+str(k), mod)
+    else:
+        dirs = os.listdir(dir_path)
+        for fil in dirs:
+            mod = random.randint(0, 511)
+            os.chmod(dir_path+"/"+fil, mod)
+    return
+
+def random_og(path):
+    u = random.randint(1025, 65536)
+    g = -1
+    os.chown(path, u, g)
+
+def chown_files(files, flen, randname, dir_path):
+    if not randname:
+        for k in range(files):
+            random_og(dir_path+"/"+"file"+str(k))
+    else:
+        dirs = os.listdir(dir_path)
+        for fil in dirs:
+            random_og(dir_path+"/"+fil)
+    return
+
+
+def chgrp_files(files, flen, randname, dir_path):
+    if not randname:
+        for k in range(files):
+            random_og(dir_path+"/"+"file"+str(k))
+    else:
+        dirs = os.listdir(dir_path)
+        for fil in dirs:
+            random_og(dir_path+"/"+fil)
+    return
+
+
+def symlink_files(files, flen, randname, dir_path):
+    try:
+        os.makedirs(dir_path+"/"+"symlink_to_files")
+    except OSError as ex:
+        if ex.errno is not errno.EEXIST:
+            raise
+    if not randname:
+        for k in range(files):
+            src_file = "file"+str(k)
+            os.symlink(dir_path+"/"+src_file,
+                       dir_path+"/"+"symlink_to_files/file"+str(k)+"_sym")
+    else:
+        dirs = os.listdir(dir_path)
+        for fil in dirs:
+            newfil = get_filename(flen)
+            os.symlink(dir_path+"/"+fil,
+                       dir_path+"/"+"symlink_to_files/"+newfil)
+    return
+
+
+def hardlink_files(files, flen, randname, dir_path):
+    try:
+        os.makedirs(dir_path+"/"+"hardlink_to_files")
+    except OSError as ex:
+        if ex.errno is not errno.EEXIST:
+            raise
+    if not randname:
+        for k in range(files):
+            src_file = "file"+str(k)
+            os.link(dir_path+"/"+src_file,
+                    dir_path+"/"+"hardlink_to_files/file"+str(k)+"_hard")
+    else:
+        dirs = os.listdir(dir_path)
+        for fil in dirs:
+            if not os.path.isdir(dir_path+"/"+fil):
+                newfil = get_filename(flen)
+                os.link(dir_path+"/"+fil,
+                        dir_path+"/"+"hardlink_to_files/"+newfil)
+    return
+
+
 def human2bytes(size):
     size_short = {
-        1024 : ['K','KB','KiB','k','kB','kiB'],
-        1024*1024 : ['M','MB','MiB'],
-        1024*1024*1024 : ['G','GB','GiB']
-}
-    num = re.search('(\d+)',size).group()
+        1024: ['K', 'KB', 'KiB', 'k', 'kB', 'kiB'],
+        1024*1024: ['M', 'MB', 'MiB'],
+        1024*1024*1024: ['G', 'GB', 'GiB']
+    }
+    num = re.search('(\d+)', size).group()
     ext = size[len(num):]
     num = int(num)
     if ext == '':
@@ -115,93 +320,270 @@ def human2bytes(size):
             size = num*value
             return size
 
-def multipledir(mnt_pnt,brdth,depth,files):
+
+def bytes2human(byts):
+    abbr = {
+        1 << 30L: "GB",
+        1 << 20L: "MB",
+        1 << 10L: "KB",
+        1: "bytes"
+    }
+    if byts == 1:
+        return '1 bytes'
+    for factor, suffix in abbr.items():
+        if byts >= factor:
+            break
+    return "%.3f %s" % (byts / factor, suffix)
+
+
+def multipledir(mnt_pnt, brdth, depth, files, fop, file_type="text",
+                inter="1000", size="100K", mins="10K", maxs="500K",
+                rand=False, l=10, randname=False):
     files_count = 1
+    size = human2bytes(size)
+    maxs = human2bytes(maxs)
+    mins = human2bytes(mins)
     for i in range(brdth):
-        breadth = mnt_pnt+"/"+str(i)
-        try:
-           os.makedirs(breadth)
-        except OSError as ex:
-            if not ex.errno is errno.EEXIST:
-                raise
-        os.chdir(breadth)
-        dir_depth = breadth
-        print breadth
+        dir_path = mnt_pnt
         for j in range(depth):
-            dir_depth = dir_depth+"/"+str(j)
+            dir_path = dir_path+"/"+"level"+str(j)+str(i)
             try:
-                os.makedirs(dir_depth)
+                os.makedirs(dir_path)
             except OSError as ex:
-                if not ex.errno is errno.EEXIST:
+                if ex.errno is not errno.EEXIST:
                     raise
-            os.chdir(dir_depth)
-            if option.file_type == "text":
-                files_count = text_files(files, files_count)
-            elif option.file_type == "sparse":
-                files_count = sparse_files(files, files_count)
-            elif option.file_type == "binary":
-                files_count = binary_files(files, files_count)
-            else:
-                print "Not a valid file type"
-                sys.exit(1)
-
-def singledir(mnt_pnt, files):
+
+            if fop == "create":
+                logger.info("Entering the directory level"+str(j)+str(i))
+                if file_type == "text":
+                    files_count = text_files(files, files_count, inter, size,
+                                             mins, maxs, rand, l, randname,
+                                             dir_path)
+                elif file_type == "sparse":
+                    files_count = sparse_files(files, files_count, inter, size,
+                                               mins, maxs, rand, l, randname,
+                                               dir_path)
+                elif file_type == "binary":
+                    files_count = binary_files(files, files_count, inter, size,
+                                               mins, maxs, rand, l, randname,
+                                               dir_path)
+                elif file_type == "tar":
+                    files_count = tar_files(files, files_count, inter, size,
+                                            mins, maxs, rand, l, randname,
+                                            dir_path)
+                else:
+                    logger.error("Not a valid file type")
+                    sys.exit(1)
+
+            elif fop == "rename":
+                logger.info("Started renaming files for the files 0 to " +
+                            str(files)+" in the directory level"+str(j) +
+                            str(i)+" ...")
+                rename_files(files, l, randname, dir_path)
+                logger.info("Finished renaming files for the files 0 to " +
+                            str(files)+" in the directory level"+str(j)+str(i))
+
+            elif fop == "chmod":
+                logger.info("Started changing permission of files for the " +
+                            "files 0 to "+str(files)+" in the directory level"
+                            + str(j)+str(i)+" ...")
+                chmod_files(files, l, randname, dir_path)
+                logger.info("Finished changing permission of files for " +
+                            "the files 0 to "+str(files) +
+                            " in the directory level"+str(j)+str(i))
+
+            elif fop == "chown":
+                logger.info("Started changing ownership of files for the " +
+                            "files 0 to " + str(files) +
+                            " in the directory level"+str(j)+str(i)+" ...")
+                chown_files(files, l, randname, dir_path)
+                logger.info("Finished changing ownership of files for " +
+                            "the files 0 to "+str(files) +
+                            " in the directory level"+str(j)+str(i))
+
+            elif fop == "chgrp":
+                logger.info("Started changing group ownership of files for " +
+                            "the files 0 to " + str(files) +
+                            " in the directory level"+str(j)+str(i)+" ...")
+                chgrp_files(files, l, randname, dir_path)
+                logger.info("Finished changing group ownership of files for " +
+                            "the files 0 to "+str(files) +
+                            " in the directory level"+str(j)+str(i))
+
+            elif fop == "symlink":
+                logger.info("Started creating symlink to the files 0 to " +
+                            str(files)+" in the directory level" +
+                            str(j)+str(i)+"...")
+                symlink_files(files, l, randname, dir_path)
+                logger.info("Finished creating symlink to the files 0 to " +
+                            str(files) + " in the directory level" +
+                            str(j)+str(i))
+
+            elif fop == "hardlink":
+                logger.info("Started creating hardlink to the files 0 to " +
+                            str(files)+" in the directory level" +
+                            str(j)+str(i)+"...")
+                hardlink_files(files, l, randname, dir_path)
+                logger.info("Finished creating hardlink to the files 0 to " +
+                            str(files) + " in the directory level" +
+                            str(j)+str(i))
+
+            elif fop == "truncate":
+                logger.info("Started truncating the files 0 to " +
+                            str(files)+" in the directory level" +
+                            str(j)+str(i)+"...")
+                truncate_files(files, mins, maxs, randname, dir_path)
+                logger.info("Finished truncating the files 0 to " +
+                            str(files)+" in the directory level" +
+                            str(j)+str(i))
+
+            elif fop == "setxattr":
+                logger.info("Started setxattr to the files 0 to " +
+                            str(files)+" in the directory level" +
+                            str(j)+str(i)+"...")
+                setxattr_files(files, randname, dir_path)
+                logger.info("Finished setxattr to the files 0 to " +
+                            str(files)+" in the directory level" +
+                            str(j)+str(i))
+
+    if fop == "create":
+        thrpt = datsiz / timr
+        logger.info("finished creating files with throughput ---- " +
+                    bytes2human(thrpt)+"ps")
+
+
+def singledir(mnt_pnt, files, fop, file_type="text", inter="1000", size="100K",
+              mins="10K", maxs="500K", rand=False, l=10, randname=False):
+
     files_count = 1
-    os.chdir(mnt_pnt)
-    if option.file_type == "text":
-        files_count = text_files(files, files_count)
-    elif option.file_type == "sparse":
-        files_count = sparse_files(files, files_count)
-    elif option.file_type == "binary":
-        files_count = binary_files(files, files_count)
-    else:
-        print "Not a valid file type"
-        sys.exit(1)
+    size = human2bytes(size)
+    maxs = human2bytes(maxs)
+    mins = human2bytes(mins)
+    if fop == "create":
+        if file_type == "text":
+            files_count = text_files(files, files_count, inter, size, mins,
+                                     maxs, rand, l, randname, mnt_pnt)
+        elif file_type == "sparse":
+            files_count = sparse_files(files, files_count, inter, size, mins,
+                                       maxs, rand, l, randname, mnt_pnt)
+        elif file_type == "binary":
+            files_count = binary_files(files, files_count, inter, size, mins,
+                                       maxs, rand, l, randname, mnt_pnt)
+        elif file_type == "tar":
+            files_count = tar_files(files, files_count, inter, size, mins,
+                                    maxs, rand, l, randname, mnt_pnt)
+        else:
+            logger.info("Not a valid file type")
+            sys.exit(1)
+        thrpt = datsiz / timr
+        logger.info("finished creating files with avg throughput ---- " +
+                    bytes2human(thrpt)+"ps")
+
+    elif fop == "rename":
+        logger.info("Started renaming files for the files 0 to " +
+                    str(files) + "...")
+        rename_files(files, l, randname, mnt_pnt)
+        logger.info("Finished renaming files for the files 0 to "+str(files))
+
+    elif fop == "chmod":
+        logger.info("Started changing permission for the files 0 to " +
+                    str(files)+" ...")
+        chmod_files(files, l, randname, mnt_pnt)
+        logger.info("Finished changing permission files for the files 0 to " +
+                    str(files))
+
+    elif fop == "chown":
+        logger.info("Started changing ownership for the files 0 to " +
+                    str(files)+"...")
+        chown_files(files, l, randname, mnt_pnt)
+        logger.info("Finished changing ownership for the files 0 to " +
+                    str(files))
+
+    elif fop == "chgrp":
+        logger.info("Started changing group ownership for the files 0 to " +
+                    str(files)+"...")
+        chgrp_files(files, l, randname, mnt_pnt)
+        logger.info("Finished changing group ownership for the files 0 to " +
+                    str(files))
+
+    elif fop == "symlink":
+        logger.info("Started creating symlink to the files 0 to " +
+                    str(files)+"...")
+        symlink_files(files, l, randname, mnt_pnt)
+        logger.info("Finished creating symlink to the files 0 to " +
+                    str(files))
+
+    elif fop == "hardlink":
+        logger.info("Started creating hardlink to the files 0 to " +
+                    str(files)+"...")
+        hardlink_files(files, l, randname, mnt_pnt)
+        logger.info("Finished creating hardlink to the files 0 to " +
+                    str(files))
+
+    elif fop == "truncate":
+        logger.info("Started truncating the files 0 to " + str(files)+"...")
+        truncate_files(files, mins, maxs, randname, mnt_pnt)
+        logger.info("Finished truncating the files 0 to " + str(files))
+
+    elif fop == "setxattr":
+        logger.info("Started setxattr to the files 0 to " + str(files)+"...")
+        setxattr_files(files, randname, mnt_pnt)
+        logger.info("Finished setxattr to the files 0 to " + str(files))
+
 
 if __name__ == '__main__':
     usage = "usage: %prog [option] <MNT_PT>"
-    parser = OptionParser(usage=usage)
-    parser.add_option("-n", dest="files",type="int" ,default=100,
-                      help="number of files in each level [default: %default]")
-    parser.add_option("--size", action = "store",type="string",
-                      help="size of the files to be used")
-    parser.add_option("--random",  action="store_true", default=True,
-                      help="random size of the file between --min and --max "
-                      "[default: %default]")
-    parser.add_option("--max", action = "store",type="string", default="500K",
-                      help="maximum size of the files, if random is True "
-                      "[default: %default]")
-    parser.add_option("--min", action = "store",type="string", default="10K",
-                      help="minimum size of the files, if random is True "
-                      "[default: %default]" )
-    parser.add_option("--single", action="store_true", dest="dir",default=True,
-                      help="create files in single directory [default: %default]" )
-    parser.add_option("--multi", action="store_false", dest="dir",
-                      help="create files in multiple directories")
-    parser.add_option("-b", dest="brdth",type="int",default=5,
-                      help="number of directories in one level(works with --multi)[default: %default]")
-    parser.add_option("-d", dest="depth",type="int",default=5,
-                      help="number of levels of directories(works with --multi)[default: %default]")
-    parser.add_option("-l", dest="flen",type="int" ,default=10,
-                      help="number of bytes for filename "
-                      "[default: %default]")
-    parser.add_option("-t","--type", action="store", type="string" , dest="file_type",default="text",
-                      help="type of the file to be created (text, sparse, binary) [default: %default]" )
-    parser.add_option("-I", dest="inter", type="int", default=100,
-                      help="print number files created of interval [defailt: %dafault]")
-    (option,args) = parser.parse_args()
-    if not args:
-        print "usage: <script> [option] <MNT_PT>"
-        print ""
-        sys.exit(1)
-    args[0] = os.path.abspath(args[0])
-    if option.size:
-        option.size = human2bytes(option.size)
-    else:
-        option.max = human2bytes(option.max)
-        option.min = human2bytes(option.min)
-    if option.dir:
-        singledir(args[0], option.files)
+    parser = argparse.ArgumentParser(formatter_class=argparse.
+                                     ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-n", dest="files", type=int, default=100,
+                        help="number of files in each level ")
+    parser.add_argument("--size", action="store", default="100k",
+                        help="size of the files to be used ")
+    parser.add_argument("--random",  action="store_true", default=False,
+                        help="random size of the file between --min and --max")
+    parser.add_argument("--max", action="store", default="500K",
+                        help="maximum size of the files, if random is True")
+    parser.add_argument("--min", action="store", default="10K",
+                        help="minimum size of the files, if random is True")
+    parser.add_argument("--single", action="store_true", dest="dir",
+                        default=True, help="create files in single directory")
+    parser.add_argument("--multi", action="store_false", dest="dir",
+                        help="create files in multiple directories")
+    parser.add_argument("-b", dest="brdth", type=int, default=5,
+                        help="number of directories in one level(works " +
+                        "with --multi) ")
+    parser.add_argument("-d", dest="depth", type=int, default=5,
+                        help="number of levels of directories  (works " +
+                        "with --multi) ")
+    parser.add_argument("-l", dest="flen", type=int, default=10,
+                        help="number of bytes for filename ( Used only when " +
+                        "randname is enabled) ")
+    parser.add_argument("-t", action="store", dest="file_type",
+                        default="text", choices=["text", "sparse", "binary",
+                                                 "tar"],
+                        help="type of the file to be created ()")
+    parser.add_argument("-I", dest="inter", type=int, default=100,
+                        help="print number files created of interval")
+    parser.add_argument("--fop", action="store", dest="fop", default="create",
+                        choices=["create", "rename", "chmod", "chown", "chgrp",
+                                 "symlink", "hardlink", "truncate",
+                                 "setxattr"],
+                        help="fop to be performed on the files")
+    parser.add_argument("-R", dest="randname", action="store_false",
+                        default=True, help="To disable random file name " +
+                        "(default: Enabled)")
+    parser.add_argument("mntpnt", help="Mount point")
+
+    args = parser.parse_args()
+    logger = setupLogger("testlost")
+    args.mntpnt = os.path.abspath(args.mntpnt)
+
+    if args.dir:
+        singledir(args.mntpnt, args.files, args.fop, args.file_type,
+                  args.inter, args.size, args.min, args.max,
+                  args.random, args.flen, args.randname)
     else:
-        multipledir(args[0], option.brdth, option.depth, option.files)
-    print "creation of files completed.\n"
+        multipledir(args.mntpnt, args.brdth, args.depth, args.files,
+                    args.fop, args.file_type, args.inter, args.size,
+                    args.min, args.max, args.random, args.flen,
+                    args.randname)
diff --git a/tests/volume.rc b/tests/volume.rc
index 59a66142e9e..f2a56888fb8 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -474,3 +474,35 @@ function mount_get_option_value {
 function get_volume_mark {
         getfattr -n trusted.glusterfs.volume-mark -ehex $1 | sed -n 's/^trusted.glusterfs.volume-mark=0x//p' | cut -b5-36 | sed 's/\([a-f0-9]\{8\}\)\([a-f0-9]\{4\}\)\([a-f0-9]\{4\}\)\([a-f0-9]\{4\}\)/\1-\2-\3-\4-/'
 }
+
+# setup geo-rep in a single a node.
+
+function setup_georep {
+
+    $CLI volume create $GMV0 replica 2  $H0:$B0/${GMV0}{1,2,3,4};
+
+    $CLI volume start $GMV0
+
+    $CLI volume create $GSV0 replica 2  $H0:$B0/${GSV0}{1,2,3,4};
+
+    $CLI volume start $GSV0
+
+    $CLI system:: execute gsec_create
+
+    $CLI volume geo-rep $GMV0 $H0::$GSV0 create push-pem
+
+    $CLI volume geo-rep $GMV0 $H0::$GSV0 start
+
+    sleep 80 # after start geo-rep takes a minute to get stable
+
+}
+
+
+# stop and delete geo-rep session
+
+function cleanup_georep {
+
+    $CLI volume geo-rep $GMV0 $H0::$GSV0 stop
+
+    $CLI volume geo-rep $GMV0 $H0::$GSV0 delete
+}