geo-rep/hook-script: Fix ssh/scp options

Always use ssh and scp with "-oPasswordAuthentication=no"
and "-oStrictHostKeyChecking=no" options. It might hang
the post script otherwise leading geo-rep setup failure

Also increased geo-rep timeout. Occasionally, it's taking
more time to reach Active/Passive status. Especially, the
first start after create.

Backport of:
 > Patch: https://review.gluster.org/20601
 > BUG: bz#1610405
 > Change-Id: I9560d64dbe0edf5db73446a9fc97dda19b88d233
 > Signed-off-by: Kotresh HR <khiremat@redhat.com>

fixes: bz#1630144
Change-Id: I9560d64dbe0edf5db73446a9fc97dda19b88d233
Signed-off-by: Kotresh HR <khiremat@redhat.com>

5 files changed, 84 insertions, 24 deletions

diff --git a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
index 83828ab0375..d8707a8fb4d 100755
--- a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
+++ b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
@@ -77,25 +77,26 @@ if [ "$val" == "" ]; then
     exit;
 fi
 SSH_PORT=`echo $val`
+SSH_OPT="-oPasswordAuthentication=no -oStrictHostKeyChecking=no"
 
 if [ -f $pub_file ]; then
     # For a non-root user copy the pub file to the user's home directory
     # For a root user copy the pub files to priv_dir->geo-rep.
     if [ "$slave_user" != "root" ]; then
-        slave_user_home_dir=`ssh -p ${SSH_PORT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"`
-        scp -P ${SSH_PORT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp
-        ssh -p ${SSH_PORT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub"
+        slave_user_home_dir=`ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"`
+        scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp
+        ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub"
     else
         if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
-            scp -P ${SSH_PORT} $pub_file $slave_ip:$pub_file_tmp
-            ssh -p ${SSH_PORT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
-            ssh -p ${SSH_PORT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
-            ssh -p ${SSH_PORT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
         else
-            scp -P ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} $pub_file $slave_ip:$pub_file_tmp
-            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
-            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
-            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            scp -P ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
         fi
     fi
 fi
diff --git a/tests/00-geo-rep/00-georep-verify-setup.t b/tests/00-geo-rep/00-georep-verify-setup.t
index f031be1898f..ea22d375157 100644
--- a/tests/00-geo-rep/00-georep-verify-setup.t
+++ b/tests/00-geo-rep/00-georep-verify-setup.t
@@ -5,7 +5,7 @@
 . $(dirname $0)/../geo-rep.rc
 . $(dirname $0)/../env.rc
 
-### Basic Tests with Distribute Replicate volumes
+SCRIPT_TIMEOUT=300
 
 ##Cleanup and start glusterd
 cleanup;
@@ -54,6 +54,30 @@ TEST glusterfs -s $H0 --volfile-id $GSV0 $M1
 #Create geo-rep session
 TEST create_georep_session $master $slave
 
+#Config gluster-command-dir
+TEST $GEOREP_CLI $master $slave config gluster-command-dir ${GLUSTER_CMD_DIR}
+
+#Config gluster-command-dir
+TEST $GEOREP_CLI $master $slave config slave-gluster-command-dir ${GLUSTER_CMD_DIR}
+
+#Enable_metavolume
+TEST $GEOREP_CLI $master $slave config use_meta_volume true
+
+#Wait for common secret pem file to be created
+EXPECT_WITHIN $GEO_REP_TIMEOUT  0 check_common_secret_file
+
+#Verify the keys are distributed
+EXPECT_WITHIN $GEO_REP_TIMEOUT  0 check_keys_distributed
+
+#Start_georep
+TEST $GEOREP_CLI $master $slave start
+
+EXPECT_WITHIN $GEO_REP_TIMEOUT  2 check_status_num_rows "Active"
+EXPECT_WITHIN $GEO_REP_TIMEOUT  2 check_status_num_rows "Passive"
+
+#Stop Geo-rep
+TEST $GEOREP_CLI $master $slave stop
+
 #Delete Geo-rep
 TEST $GEOREP_CLI $master $slave delete
 
diff --git a/tests/00-geo-rep/georep-basic-dr-rsync.t b/tests/00-geo-rep/georep-basic-dr-rsync.t
index d083a96a2e2..7c8499fa5d9 100644
--- a/tests/00-geo-rep/georep-basic-dr-rsync.t
+++ b/tests/00-geo-rep/georep-basic-dr-rsync.t
@@ -5,6 +5,8 @@
 . $(dirname $0)/../geo-rep.rc
 . $(dirname $0)/../env.rc
 
+SCRIPT_TIMEOUT=500
+
 AREQUAL_PATH=$(dirname $0)/../utils
 test "`uname -s`" != "Linux" && {
     CFLAGS="$CFLAGS -I$(dirname $0)/../../../contrib/argp-standalone ";
@@ -42,6 +44,10 @@ TEST $CLI volume start $GMV0
 ##create_and_start_slave_volume
 TEST $CLI volume create $GSV0 replica 2 $H0:$B0/${GSV0}{1,2,3,4};
 TEST $CLI volume start $GSV0
+TEST $CLI volume set $GSV0 performance.stat-prefetch off
+TEST $CLI volume set $GSV0 performance.quick-read off
+TEST $CLI volume set $GSV0 performance.readdir-ahead off
+TEST $CLI volume set $GSV0 performance.read-ahead off
 
 ##Create, start and mount meta_volume
 TEST $CLI volume create $META_VOL replica 3 $H0:$B0/${META_VOL}{1,2,3};
@@ -76,6 +82,12 @@ TEST $GEOREP_CLI $master $slave config use_meta_volume true
 #Set changelog roll-over time to 3 secs
 TEST $CLI volume set $GMV0 changelog.rollover-time 3
 
+#Wait for common secret pem file to be created
+EXPECT_WITHIN $GEO_REP_TIMEOUT  0 check_common_secret_file
+
+#Verify the keys are distributed
+EXPECT_WITHIN $GEO_REP_TIMEOUT  0 check_keys_distributed
+
 #Start_georep
 TEST $GEOREP_CLI $master $slave start
 
@@ -84,9 +96,9 @@ EXPECT_WITHIN $GEO_REP_TIMEOUT  2 check_status_num_rows "Passive"
 
 #data_tests "hybrid"
 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 regular_file_ok ${slave_mnt}/hybrid_f1
-EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/$hybrid_d1
-EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/hybrid_f3 ${slave_mnt}/hybrid_f4
-EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/hybrid_d3 ${slave_mnt}/hybrid_d4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/hybrid_d1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_file_ok ${slave_mnt}/hybrid_f3 ${slave_mnt}/hybrid_f4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_dir_ok ${slave_mnt}/hybrid_d3 ${slave_mnt}/hybrid_d4
 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 symlink_ok hybrid_f1 ${slave_mnt}/hybrid_sl1
 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 hardlink_file_ok ${slave_mnt}/hybrid_f1 ${slave_mnt}/hybrid_hl1
 EXPECT_WITHIN $GEO_REP_TIMEOUT 1 unlink_ok ${slave_mnt}/hybrid_f2
diff --git a/tests/00-geo-rep/georep-basic-dr-tarssh.t b/tests/00-geo-rep/georep-basic-dr-tarssh.t
index 1163b8d842a..cbcad80ba70 100644
--- a/tests/00-geo-rep/georep-basic-dr-tarssh.t
+++ b/tests/00-geo-rep/georep-basic-dr-tarssh.t
@@ -3,6 +3,9 @@
 . $(dirname $0)/../include.rc
 . $(dirname $0)/../volume.rc
 . $(dirname $0)/../geo-rep.rc
+. $(dirname $0)/../env.rc
+
+SCRIPT_TIMEOUT=500
 
 AREQUAL_PATH=$(dirname $0)/../utils
 test "`uname -s`" != "Linux" && {
@@ -41,6 +44,10 @@ TEST $CLI volume start $GMV0
 ##create_and_start_slave_volume
 TEST $CLI volume create $GSV0 replica 2 $H0:$B0/${GSV0}{1,2,3,4};
 TEST $CLI volume start $GSV0
+TEST $CLI volume set $GSV0 performance.stat-prefetch off
+TEST $CLI volume set $GSV0 performance.quick-read off
+TEST $CLI volume set $GSV0 performance.readdir-ahead off
+TEST $CLI volume set $GSV0 performance.read-ahead off
 
 ##Create, start and mount meta_volume
 TEST $CLI volume create $META_VOL replica 3 $H0:$B0/${META_VOL}{1,2,3};
@@ -78,6 +85,12 @@ TEST $CLI volume set $GMV0 changelog.rollover-time 3
 #Config tarssh as sync-engine
 TEST $GEOREP_CLI $master $slave config use_tarssh true
 
+#Wait for common secret pem file to be created
+EXPECT_WITHIN $GEO_REP_TIMEOUT  0 check_common_secret_file
+
+#Verify the keys are distributed
+EXPECT_WITHIN $GEO_REP_TIMEOUT  0 check_keys_distributed
+
 #Start_georep
 TEST $GEOREP_CLI $master $slave start
 
@@ -86,9 +99,9 @@ EXPECT_WITHIN $GEO_REP_TIMEOUT  2 check_status_num_rows "Passive"
 
 #data_tests "hybrid"
 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 regular_file_ok ${slave_mnt}/hybrid_f1
-EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/$hybrid_d1
-EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/hybrid_f3 ${slave_mnt}/hybrid_f4
-EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/hybrid_d3 ${slave_mnt}/hybrid_d4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/hybrid_d1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_file_ok ${slave_mnt}/hybrid_f3 ${slave_mnt}/hybrid_f4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_dir_ok ${slave_mnt}/hybrid_d3 ${slave_mnt}/hybrid_d4
 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 symlink_ok hybrid_f1 ${slave_mnt}/hybrid_sl1
 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 hardlink_file_ok ${slave_mnt}/hybrid_f1 ${slave_mnt}/hybrid_hl1
 EXPECT_WITHIN $GEO_REP_TIMEOUT 1 unlink_ok ${slave_mnt}/hybrid_f2
diff --git a/tests/geo-rep.rc b/tests/geo-rep.rc
index cdad4f1cb62..8c5391dbfa9 100644
--- a/tests/geo-rep.rc
+++ b/tests/geo-rep.rc
@@ -3,7 +3,20 @@ GEO_REP_TIMEOUT=60
 function check_status_num_rows()
 {
     local search_key=$1
-    $GEOREP_CLI $master $slave status detail | egrep -i "$search_key" | wc -l
+    $GEOREP_CLI $master $slave status | grep -F "$search_key" | wc -l
+}
+
+function check_keys_distributed()
+{
+    local search_key=$(cat /var/lib/glusterd/geo-replication/master_slave_common_secret.pem.pub)
+    grep -F "$search_key" ~/.ssh/authorized_keys > /dev/null
+    echo $?
+}
+
+function check_common_secret_file()
+{
+    stat /var/lib/glusterd/geo-replication/master_slave_common_secret.pem.pub
+    echo $?
 }
 
 function create_data()
@@ -22,8 +35,8 @@ function create_data()
     # GF_FOP_CREATE
     # GF_FOP_SETATTR
 
-    # Regular file
-    touch ${master_mnt}/${prefix}_f1
+    # Regular file + data
+    echo "HelloWorld!" > ${master_mnt}/${prefix}_f1
     touch ${master_mnt}/${prefix}_f2
     touch ${master_mnt}/${prefix}_f3
 
@@ -40,9 +53,6 @@ function create_data()
     ln -s ${prefix}_f1 ${prefix}_sl1
     cd -
 
-    # data
-    echo "HelloWorld!" >> ${master_mnt}/${prefix}_f1
-
     # UNLINK
     rm ${master_mnt}/${prefix}_f2