glusterd: rebalance support for cluster.rc framework

Issue:Rebalance is failing in cluster framework (any simulated cluster environment in same node ).
RCA:
  1. we are passing always "localhost" as volfile server for rebalance xlator .
  2. Rebalance daemons are overwriting  unix socket and log files each other.
     (All rebalance processes are creating socket with same name) .

Fix: set vol_file_server, unix socket and log files properly.

Change-Id: I6654461e00c2a164b2f1f1db24a316c4180dd8d5
BUG: 1231437
Signed-off-by: anand <anekkunt@redhat.com>
Reviewed-on: http://review.gluster.org/11210
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

3 files changed, 67 insertions, 1 deletions

diff --git a/tests/bugs/glusterd/bug-1231437-rebalance-test-in-cluster.t b/tests/bugs/glusterd/bug-1231437-rebalance-test-in-cluster.t
new file mode 100644
index 00000000000..3257f6994dd
--- /dev/null
+++ b/tests/bugs/glusterd/bug-1231437-rebalance-test-in-cluster.t
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../cluster.rc
+. $(dirname $0)/../../volume.rc
+
+
+cleanup;
+TEST launch_cluster 2;
+TEST $CLI_1 peer probe $H2;
+
+EXPECT_WITHIN $PROBE_TIMEOUT 1 peer_count
+
+$CLI_1 volume create $V0 $H1:$B1/$V0  $H2:$B2/$V0
+EXPECT 'Created' cluster_volinfo_field 1 $V0 'Status';
+
+$CLI_1 volume start $V0
+EXPECT 'Started' cluster_volinfo_field 1 $V0 'Status';
+
+#Mount FUSE
+TEST glusterfs -s $H1 --volfile-id=$V0 $M0;
+
+TEST mkdir $M0/dir{1..4};
+TEST touch $M0/dir{1..4}/files{1..4};
+
+TEST $CLI_1 volume add-brick $V0 $H1:$B1/${V0}1 $H2:$B2/${V0}1
+
+TEST $CLI_1 volume rebalance $V0  start
+EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" cluster_rebalance_status_field 1  $V0
+
+cleanup;
diff --git a/tests/cluster.rc b/tests/cluster.rc
index b734f9d4501..6fd74bc8ba4 100644
--- a/tests/cluster.rc
+++ b/tests/cluster.rc
@@ -138,3 +138,19 @@ function define_clis() {
 function peer_count() {
     $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
 }
+
+function cluster_rebalance_status_field {
+        #The rebalance status can be upto 3 words, (ex:'fix-layout in progress'), hence the awk-print $7 thru $9.
+        #But if the status is less than 3 words, it also prints the next field i.e the run_time_in_secs.(ex:'completed 3.00').
+        #So we trim the numbers out with  `tr`. Finally remove the trailing white spaces with sed. What we get is one of the
+        #strings in the 'cli_vol_task_status_str' char array of cli-rpc-ops.c
+
+        eval  \$CLI_$1 volume rebalance $2 status | awk '{print $7,$8,$9}' |sed -n 3p |tr -d '[^0-9+\.]'|sed 's/ *$//g'
+}
+
+function cluster_volinfo_field()
+{
+    local vol=$2;
+    local field=$3;
+    eval  \$CLI_$1  volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 03cc53c6cce..c4674f56c33 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -193,6 +193,7 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
         char                   logfile[PATH_MAX] = {0,};
         char                   volname[PATH_MAX] = {0,};
         char                   valgrind_logfile[PATH_MAX] = {0,};
+        char                   *volfileserver = NULL;
 
         priv    = THIS->private;
 
@@ -253,8 +254,26 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
         }
 
         snprintf (volname, sizeof(volname), "rebalance/%s", volinfo->volname);
+
+        if (dict_get_str (THIS->options, "transport.socket.bind-address",
+                          &volfileserver) == 0) {
+               /*In the case of running multiple glusterds on a single machine,
+                *we should ensure that log file and unix socket file shouls be
+                *unique in given cluster */
+
+                GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD (sockfile, volinfo,
+                                                   priv);
+                snprintf (logfile, PATH_MAX, "%s/%s-%s-%s.log",
+                          DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname,
+                          (cmd == GF_DEFRAG_CMD_START_TIER ? "tier":"rebalance"),
+                          uuid_utoa(MY_UUID));
+
+        } else {
+                volfileserver = "localhost";
+        }
+
         runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", "localhost", "--volfile-id", volname,
+                         "-s", volfileserver, "--volfile-id", volname,
                          "--xlator-option", "*dht.use-readdirp=yes",
                          "--xlator-option", "*dht.lookup-unhashed=yes",
                          "--xlator-option", "*dht.assert-no-child-down=yes",