glusterd: Protected conf->xprt_list racy access.

- epoll on RPCSVC_EVENT_ACCEPT would add corresponding xprt
  onto the xprt_list. Concurrently, synctask thread (volume op)
  would call into glusterd_fetchspec_notify which iterates on
  the xprt_list. Added a mutex to protect such a racy access of
  the list.

Change-Id: Idc51b4bdb1c814dfab7790e1c899d6977f7640f2
BUG: 878873
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/4241
Reviewed-by: Raghavendra G <raghavendra@gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>

3 files changed, 43 insertions, 3 deletions

diff --git a/tests/bugs/bug-878873.t b/tests/bugs/bug-878873.t
new file mode 100644
index 00000000..efdcaca8
--- /dev/null
+++ b/tests/bugs/bug-878873.t
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}2 $H0:$B0/${V0}3;
+
+function brick_count()
+{
+    local vol=$1;
+
+    $CLI volume info $vol | egrep "^Brick[0-9]+: " | wc -l;
+}
+
+
+TEST $CLI volume start $V0
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2 start;
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2 commit;
+EXPECT '2' brick_count $V0
+
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}3;
+EXPECT '1' brick_count $V0
+
+
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 02dd5fac..bd9e6e03 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -156,10 +156,15 @@ glusterd_fetchspec_notify (xlator_t *this)
 
         priv = this->private;
 
-        list_for_each_entry (trans, &priv->xprt_list, list) {
-                rpcsvc_callback_submit (priv->rpc, trans, &glusterd_cbk_prog,
-                                        GF_CBK_FETCHSPEC, NULL, 0);
+        pthread_mutex_lock (&priv->xprt_lock);
+        {
+                list_for_each_entry (trans, &priv->xprt_list, list) {
+                        rpcsvc_callback_submit (priv->rpc, trans,
+                                                &glusterd_cbk_prog,
+                                                GF_CBK_FETCHSPEC, NULL, 0);
+                }
         }
+        pthread_mutex_unlock (&priv->xprt_lock);
 
         ret = 0;
 
@@ -217,12 +222,16 @@ glusterd_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
         {
                 INIT_LIST_HEAD (&xprt->list);
 
+                pthread_mutex_lock (&priv->xprt_lock);
                 list_add_tail (&xprt->list, &priv->xprt_list);
+                pthread_mutex_unlock (&priv->xprt_lock);
                 break;
         }
         case RPCSVC_EVENT_DISCONNECT:
         {
+                pthread_mutex_lock (&priv->xprt_lock);
                 list_del (&xprt->list);
+                pthread_mutex_unlock (&priv->xprt_lock);
                 pmap_registry_remove (this, 0, NULL, GF_PMAP_PORT_NONE, xprt);
                 break;
         }
@@ -995,6 +1004,7 @@ init (xlator_t *this)
         conf->gfs_mgmt = &gd_brick_prog;
         strncpy (conf->workdir, workdir, PATH_MAX);
 
+        pthread_mutex_init (&conf->xprt_lock, NULL);
         INIT_LIST_HEAD (&conf->xprt_list);
 
         glusterd_friend_sm_init ();
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index c2041ad3..fddfabd3 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -135,6 +135,7 @@ typedef struct {
         nodesrv_t         *nfs;
         struct pmap_registry *pmap;
         struct list_head  volumes;
+        pthread_mutex_t   xprt_lock;
         struct list_head  xprt_list;
         glusterd_store_handle_t *handle;
         gf_timer_t *timer;