core: run many bricks within one glusterfsd process

This patch adds support for multiple brick translator stacks running
in a single brick server process.  This reduces our per-brick memory usage by
approximately 3x, and our appetite for TCP ports even more.  It also creates
potential to avoid process/thread thrashing, and to improve QoS by scheduling
more carefully across the bricks, but realizing that potential will require
further work.

Multiplexing is controlled by the "cluster.brick-multiplex" global option.  By
default it's off, and bricks are started in separate processes as before.  If
multiplexing is enabled, then *compatible* bricks (mostly those with the same
transport options) will be started in the same process.

Change-Id: I45059454e51d6f4cbb29a4953359c09a408695cb
BUG: 1385758
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: https://review.gluster.org/14763
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

3 files changed, 27 insertions, 9 deletions

diff --git a/tests/bugs/glusterd/bug-1245045-remove-brick-validation.t b/tests/bugs/glusterd/bug-1245045-remove-brick-validation.t
index 22a8d557d28..597c40ca4ec 100644
--- a/tests/bugs/glusterd/bug-1245045-remove-brick-validation.t
+++ b/tests/bugs/glusterd/bug-1245045-remove-brick-validation.t
@@ -19,6 +19,7 @@ kill_glusterd 2
 TEST ! $CLI_1 volume remove-brick $V0 $H2:$B2/${V0} start
 
 TEST start_glusterd 2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V0 $H2 $B2/${V0}
 
 EXPECT_WITHIN $PROBE_TIMEOUT 2 peer_count
 
@@ -33,6 +34,7 @@ kill_glusterd 2
 TEST ! $CLI_1 volume remove-brick $V0 $H2:$B2/${V0} commit
 
 TEST start_glusterd 2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V0 $H2 $B2/${V0}
 
 EXPECT_WITHIN $PROBE_TIMEOUT 2 peer_count
 
diff --git a/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t b/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
index 19defe435c1..afbc30264e4 100644
--- a/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
+++ b/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
@@ -20,14 +20,26 @@ function create_dist_tier_vol () {
 }
 
 function non_zero_check () {
-if [ "$1" -ne 0 ]
-then
-        echo "0"
-else
-        echo "1"
-fi
+        if [ "$1" -ne 0 ]
+        then
+                echo "0"
+        else
+                echo "1"
+        fi
 }
 
+function num_bricks_up {
+        local b
+        local n_up=0
+
+        for b in $B0/hot/${V0}{1..2} $B0/cold/${V0}{1..3}; do
+                if [ x"$(brick_up_status $V0 $H0 $b)" = x"1" ]; then
+                        n_up=$((n_up+1))
+                fi
+        done
+
+        echo $n_up
+}
 
 cleanup;
 
@@ -39,6 +51,8 @@ TEST $CLI volume status
 
 #Create and start a tiered volume
 create_dist_tier_vol
+# Wait for the bricks to come up, *then* the tier daemon.
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 5 num_bricks_up
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_daemon_check
 sleep 5   #wait for some time to run tier daemon
 time_before_restarting=$(rebalance_run_time $V0);
@@ -51,6 +65,8 @@ EXPECT "0" non_zero_check $time_before_restarting;
 kill -9 $(pidof glusterd);
 TEST glusterd;
 sleep 2;
+# Wait for the bricks to come up, *then* the tier daemon.
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 5 num_bricks_up
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_daemon_check;
 time1=$(rebalance_run_time $V0);
 EXPECT "0" non_zero_check $time1;
diff --git a/tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t b/tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t
index 7f2f3cc66ca..34959f5b0c6 100644
--- a/tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t
+++ b/tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t
@@ -30,7 +30,7 @@ TEST kill_glusterd 2
 TEST kill_glusterd 3
 
 # Server quorum is not met. Brick on 1st node must be down
-EXPECT "0" brick_up_status_1 $V0 $H1 $B1/${V0}1
+EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "0" brick_up_status_1 $V0 $H1 $B1/${V0}1
 
 # Set quorum ratio 95. means 95 % or more than 95% nodes of total available node
 # should be available for performing volume operation.
@@ -46,8 +46,8 @@ TEST $glusterd_2
 EXPECT_WITHIN $PROBE_TIMEOUT 1 peer_count
 
 # Server quorum is still not met. Bricks should be down on 1st and 2nd nodes
-EXPECT "0" brick_up_status_1 $V0 $H1 $B1/${V0}1
-EXPECT "0" brick_up_status_1 $V0 $H2 $B2/${V0}2
+EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "0" brick_up_status_1 $V0 $H1 $B1/${V0}1
+EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "0" brick_up_status_1 $V0 $H2 $B2/${V0}2
 
 # Bring back 3rd glusterd
 TEST $glusterd_3