event/epoll: Add back socket for polling of events immediately after

             reading the entire rpc message from the wire

Currently socket is added back for future events after higher layers
(rpc, xlators etc) have processed the message. If message processing
involves signficant delay (as in writev replies processed by Erasure
Coding), performance takes hit. Hence this patch modifies
transport/socket to add back the socket for polling of events
immediately after reading the entire rpc message, but before
notification to higher layers.

credits: Thanks to "Kotresh Hiremath Ravishankar"
         <khiremat@redhat.com> for assitance in fixing a regression in
         bitrot caused by this patch.

Change-Id: I04b6b9d0b51a1cfb86ecac3c3d87a5f388cf5800
BUG: 1448364
Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: https://review.gluster.org/15036
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Amar Tumballi <amarts@redhat.com>

1 files changed, 69 insertions, 53 deletions

diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 6db2a0fca3f..9b76c7576d4 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1732,8 +1732,7 @@ out:
 
 /* XXX: move these into @ctx */
 static char *oldvolfile = NULL;
-static int oldvollen = 0;
-
+static int   oldvollen;
 
 
 int
@@ -1743,7 +1742,7 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         gf_getspec_rsp           rsp   = {0,};
         call_frame_t            *frame = NULL;
         glusterfs_ctx_t         *ctx = NULL;
-        int                      ret   = 0;
+        int                      ret   = 0, locked = 0;
         ssize_t                  size = 0;
         FILE                    *tmpfp = NULL;
         char                    *volfilebuf = NULL;
@@ -1773,74 +1772,85 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         ret = 0;
         size = rsp.op_ret;
 
-        if (size == oldvollen && (memcmp (oldvolfile, rsp.spec, size) == 0)) {
-                gf_log (frame->this->name, GF_LOG_INFO,
-                        "No change in volfile, continuing");
-                goto out;
-        }
+        LOCK (&ctx->volfile_lock);
+        {
+                locked = 1;
 
-        tmpfp = tmpfile ();
-        if (!tmpfp) {
-                ret = -1;
-                goto out;
-        }
+                if (size == oldvollen && (memcmp (oldvolfile, rsp.spec, size) == 0)) {
+                        gf_log (frame->this->name, GF_LOG_INFO,
+                                "No change in volfile, continuing");
+                        goto out;
+                }
 
-        fwrite (rsp.spec, size, 1, tmpfp);
-        fflush (tmpfp);
-        if (ferror (tmpfp)) {
-                ret = -1;
-                goto out;
-        }
+                tmpfp = tmpfile ();
+                if (!tmpfp) {
+                        ret = -1;
+                        goto out;
+                }
+
+                fwrite (rsp.spec, size, 1, tmpfp);
+                fflush (tmpfp);
+                if (ferror (tmpfp)) {
+                        ret = -1;
+                        goto out;
+                }
+
+                /*  Check if only options have changed. No need to reload the
+                 *  volfile if topology hasn't changed.
+                 *  glusterfs_volfile_reconfigure returns 3 possible return states
+                 *  return 0          =======> reconfiguration of options has succeeded
+                 *  return 1          =======> the graph has to be reconstructed and all the xlators should be inited
+                 *  return -1(or -ve) =======> Some Internal Error occurred during the operation
+                 */
+
+                ret = glusterfs_volfile_reconfigure (oldvollen, tmpfp, ctx, oldvolfile);
+                if (ret == 0) {
+                        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+                                "No need to re-load volfile, reconfigure done");
+                        if (oldvolfile)
+                                volfilebuf = GF_REALLOC (oldvolfile, size);
+                        else
+                                volfilebuf = GF_CALLOC (1, size, gf_common_mt_char);
+                        if (!volfilebuf) {
+                                ret = -1;
+                                goto out;
+                        }
+                        oldvolfile = volfilebuf;
+                        oldvollen = size;
+                        memcpy (oldvolfile, rsp.spec, size);
+                        goto out;
+                }
 
-        /*  Check if only options have changed. No need to reload the
-        *  volfile if topology hasn't changed.
-        *  glusterfs_volfile_reconfigure returns 3 possible return states
-        *  return 0          =======> reconfiguration of options has succeeded
-        *  return 1          =======> the graph has to be reconstructed and all the xlators should be inited
-        *  return -1(or -ve) =======> Some Internal Error occurred during the operation
-        */
+                if (ret < 0) {
+                        gf_log ("glusterfsd-mgmt",
+                                GF_LOG_DEBUG, "Reconfigure failed !!");
+                        goto out;
+                }
+
+                ret = glusterfs_process_volfp (ctx, tmpfp);
+                /* tmpfp closed */
+                tmpfp = NULL;
+                if (ret)
+                        goto out;
 
-        ret = glusterfs_volfile_reconfigure (oldvollen, tmpfp, ctx, oldvolfile);
-        if (ret == 0) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "No need to re-load volfile, reconfigure done");
                 if (oldvolfile)
                         volfilebuf = GF_REALLOC (oldvolfile, size);
                 else
                         volfilebuf = GF_CALLOC (1, size, gf_common_mt_char);
+
                 if (!volfilebuf) {
                         ret = -1;
                         goto out;
                 }
+
                 oldvolfile = volfilebuf;
                 oldvollen = size;
                 memcpy (oldvolfile, rsp.spec, size);
-                goto out;
         }
+        UNLOCK (&ctx->volfile_lock);
 
-        if (ret < 0) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG, "Reconfigure failed !!");
-                goto out;
-        }
-
-        ret = glusterfs_process_volfp (ctx, tmpfp);
-        /* tmpfp closed */
-        tmpfp = NULL;
-        if (ret)
-                goto out;
-
-        if (oldvolfile)
-                volfilebuf = GF_REALLOC (oldvolfile, size);
-        else
-                volfilebuf = GF_CALLOC (1, size, gf_common_mt_char);
+        locked = 0;
 
-        if (!volfilebuf) {
-                ret = -1;
-                goto out;
-        }
-        oldvolfile = volfilebuf;
-        oldvollen = size;
-        memcpy (oldvolfile, rsp.spec, size);
         if (!is_mgmt_rpc_reconnect) {
                 need_emancipate = 1;
                 glusterfs_mgmt_pmap_signin (ctx);
@@ -1848,6 +1858,10 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
 out:
+
+        if (locked)
+                UNLOCK (&ctx->volfile_lock);
+
         STACK_DESTROY (frame->root);
 
         free (rsp.spec);
@@ -2345,6 +2359,8 @@ glusterfs_mgmt_init (glusterfs_ctx_t *ctx)
         if (ctx->mgmt)
                 return 0;
 
+        LOCK_INIT (&ctx->volfile_lock);
+
         if (cmd_args->volfile_server_port)
                 port = cmd_args->volfile_server_port;