glusterfsd : newly added brick receives fops only after it is started

When new bricks are added in the middle of an on-going
fop like 'rm', the volfile changes without waiting for
the newly added bricks to get port. Fops are sent to all
bricks and may fail on some with ENOTCONN as these bricks
may not have a port yet.

This patch ensures that the volfile change happens only
after all the bricks have a port.


Change-Id: I7ed2413475f80d0cc8849fed33036ade8d75a191
BUG: 1233151
Signed-off-by: Sakshi <sabansal@redhat.com>
Reviewed-on: http://review.gluster.org/11342
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Tested-by: Atin Mukherjee <amukherj@redhat.com>

3 files changed, 27 insertions, 4 deletions

diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 459d79fea0e..d7e87728e74 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -33,6 +33,7 @@
 #include "xlator.h"
 
 static gf_boolean_t is_mgmt_rpc_reconnect = _gf_false;
+int need_emancipate = 0;
 
 int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
 int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
@@ -1571,6 +1572,7 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         oldvollen = size;
         memcpy (oldvolfile, rsp.spec, size);
         if (!is_mgmt_rpc_reconnect) {
+                need_emancipate = 1;
                 glusterfs_mgmt_pmap_signin (ctx);
                 is_mgmt_rpc_reconnect =  _gf_true;
         }
@@ -1580,9 +1582,6 @@ out:
 
         free (rsp.spec);
 
-        if (ctx)
-                emancipate (ctx, ret);
-
         // Stop if server is running at an unsupported op-version
         if (ENOTSUP == ret) {
                 gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
@@ -2070,12 +2069,15 @@ mgmt_pmap_signin2_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         pmap_signin_rsp  rsp   = {0,};
+        glusterfs_ctx_t *ctx   = NULL;
         call_frame_t    *frame = NULL;
         int              ret   = 0;
 
+        ctx = glusterfsd_ctx;
         frame = myframe;
 
         if (-1 == req->rpc_status) {
+                ret = -1;
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
                 goto out;
@@ -2092,9 +2094,13 @@ mgmt_pmap_signin2_cbk (struct rpc_req *req, struct iovec *iov, int count,
         if (-1 == rsp.op_ret) {
                 gf_log (frame->this->name, GF_LOG_ERROR,
                         "failed to register the port with glusterd");
+                ret = -1;
                 goto out;
         }
 out:
+        if (need_emancipate)
+                emancipate (ctx, ret);
+
         STACK_DESTROY (frame->root);
         return 0;
 
@@ -2107,6 +2113,7 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
         pmap_signin_rsp  rsp   = {0,};
         call_frame_t    *frame = NULL;
         int              ret   = 0;
+        int              emancipate_ret  = -1;
         pmap_signin_req  pmap_req = {0, };
         cmd_args_t      *cmd_args = NULL;
         glusterfs_ctx_t *ctx      = NULL;
@@ -2115,6 +2122,7 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
         frame = myframe;
 
         if (-1 == req->rpc_status) {
+                ret = -1;
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
                 goto out;
@@ -2131,6 +2139,7 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
         if (-1 == rsp.op_ret) {
                 gf_log (frame->this->name, GF_LOG_ERROR,
                         "failed to register the port with glusterd");
+                ret = -1;
                 goto out;
         }
 
@@ -2139,6 +2148,7 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
 
         if (!cmd_args->brick_port2) {
                 /* We are done with signin process */
+                emancipate_ret = 0;
                 goto out;
         }
 
@@ -2155,6 +2165,8 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
         return 0;
 
 out:
+        if (need_emancipate && (ret < 0 || !cmd_args->brick_port2))
+                emancipate (ctx, emancipate_ret);
 
         STACK_DESTROY (frame->root);
         return 0;
@@ -2166,6 +2178,7 @@ glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
         call_frame_t     *frame                 = NULL;
         pmap_signin_req   req                   = {0, };
         int               ret                   = -1;
+        int               emancipate_ret        = -1;
         cmd_args_t       *cmd_args              = NULL;
         char              brick_name[PATH_MAX]  = {0,};
 
@@ -2175,6 +2188,7 @@ glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
         if (!cmd_args->brick_port || !cmd_args->brick_name) {
                 gf_log ("fsd-mgmt", GF_LOG_DEBUG,
                         "portmapper signin arguments not given");
+                emancipate_ret = 0;
                 goto out;
         }
 
@@ -2193,6 +2207,8 @@ glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
                                    (xdrproc_t)xdr_pmap_signin_req);
 
 out:
+        if (need_emancipate && ret < 0)
+                emancipate (ctx, emancipate_ret);
         return ret;
 }
 
diff --git a/tests/bugs/fuse/bug-1126048.c b/tests/bugs/fuse/bug-1126048.c
index 60329955258..01c3484172d 100644
--- a/tests/bugs/fuse/bug-1126048.c
+++ b/tests/bugs/fuse/bug-1126048.c
@@ -17,6 +17,7 @@ main (int argc, char **argv)
         int ret = 0;
         int fd = 0;
         char    *cmd = argv[1];
+        struct stat stbuf = {0, };
 
         printf ("cmd is: %s\n", cmd);
         fd = open("a.txt", O_CREAT|O_RDWR, 0644);
@@ -30,6 +31,9 @@ main (int argc, char **argv)
                 printf ("Not able to print %s\n", strerror (errno));
         system(cmd);
         sleep(1); /* No way to confirm graph switch so sleep 1 */
+        ret = fstat (fd, &stbuf);
+        if (ret < 0)
+                printf ("fstat failed %\n", strerror (errno));
         ret = fsync(fd);
         if (ret < 0)
                 printf ("Not able to fsync %s\n", strerror (errno));
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index b14566aa1bb..8772dd1d108 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1318,7 +1318,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         volinfo->subvol_count = (volinfo->brick_count /
                                  volinfo->dist_leaf_count);
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        ret = glusterd_create_volfiles (volinfo);
         if (ret)
                 goto out;
 
@@ -1396,6 +1396,9 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                               _glusterd_restart_gsync_session, &param);
         }
         volinfo->caps = caps;
+
+        ret = glusterd_fetchspec_notify (this);
+
 out:
         GF_FREE (free_ptr1);
         GF_FREE (free_ptr2);