glusterd : check if all bricks are started before performing remove-brick

Change-Id: Ie9e24e037b7a39b239a7badb983504963d664324
BUG: 1225716
Signed-off-by: Sakshi <sabansal@redhat.com>
Reviewed-on: http://review.gluster.org/10954
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

5 files changed, 54 insertions, 1 deletions

diff --git a/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t b/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t
new file mode 100644
index 00000000000..657c7f8eb01
--- /dev/null
+++ b/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume start $V0
+
+#kill a brick process
+kill -15 `cat $GLUSTERD_WORKDIR/vols/$V0/run/$H0-d-backends-${V0}1.pid`;
+
+#remove-brick start should fail as the brick is down
+TEST ! $CLI volume remove-brick $V0 $H0:$B0/${V0}1 start
+
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" brick_up_status $V0 $H0 $B0/${V0}1
+
+#remove-brick start should succeed as the brick is up
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}1 start
+
+#kill a brick process
+kill -15 `cat $GLUSTERD_WORKDIR/vols/$V0/run/$H0-d-backends-${V0}1.pid`;
+
+#remove-brick commit should pass even if the brick is down
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}1 commit
+
+cleanup;
diff --git a/tests/bugs/glusterd/bug-857330/normal.t b/tests/bugs/glusterd/bug-857330/normal.t
index c575038b840..d42f5dacaf6 100755
--- a/tests/bugs/glusterd/bug-857330/normal.t
+++ b/tests/bugs/glusterd/bug-857330/normal.t
@@ -46,6 +46,8 @@ TEST $CLI volume replace-brick $V0 $REP_BRICK_PAIR commit force;
 ##################
 ## Remove-brick ##
 ##################
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" brick_up_status $V0 $H0 $B0/${V0}3
+
 COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 start"
 PATTERN="ID:"
 TEST check-and-store-task-id
diff --git a/tests/bugs/glusterd/bug-857330/xml.t b/tests/bugs/glusterd/bug-857330/xml.t
index c25a8a63251..6b87f21d9c1 100755
--- a/tests/bugs/glusterd/bug-857330/xml.t
+++ b/tests/bugs/glusterd/bug-857330/xml.t
@@ -52,6 +52,8 @@ TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}2 $H0:$B0/${V0}3 commit force
 ##################
 ## Remove-brick ##
 ##################
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" brick_up_status $V0 $H0 $B0/${V0}3
+
 COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 start"
 PATTERN="task-id"
 TEST check-and-store-task-id-xml
diff --git a/tests/volume.rc b/tests/volume.rc
index 11ef9939d4d..36336266d5c 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -19,6 +19,14 @@ function online_brick_count ()
     pgrep glusterfsd | wc -l
 }
 
+function brick_up_status {
+        local vol=$1
+        local host=$2
+        local brick=$3
+        brick_pid=$(get_brick_pid $vol $host $brick)
+        gluster volume status | grep $brick_pid | awk '{print $4}'
+}
+
 function volume_option()
 {
         local vol=$1
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index 8b69924f9d7..b14566aa1bb 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1748,8 +1748,17 @@ glusterd_remove_brick_validate_bricks (gf1_op_commands cmd, int32_t brick_count,
                         }
                 }
 
-                if (glusterd_is_local_brick (THIS, volinfo, brickinfo))
+                if (glusterd_is_local_brick (THIS, volinfo, brickinfo)) {
+                        if (cmd == GF_OP_CMD_START &&
+                            brickinfo->status != GF_BRICK_STARTED) {
+                                snprintf (msg, sizeof (msg), "Found stopped "
+                                          "brick %s", brick);
+                                *errstr = gf_strdup (msg);
+                                ret = -1;
+                                goto out;
+                        }
                         continue;
+                }
 
                 rcu_read_lock ();
                 peerinfo = glusterd_peerinfo_find_by_uuid