gNFS: Set default outstanding RPC limit to 16

With 64, NFS server hangs with large I/O load (~ 64 threads writing
to NFS server). The test results from Ben England (Performance expert)
suggest to set it as 16 instead of 64.

Change-Id: I418ff5ba0a3e9fdb14f395b8736438ee1bbd95f4
BUG: 1008301
Signed-off-by: Santosh Kumar Pradhan <spradhan@redhat.com>
Reviewed-on: http://review.gluster.org/6696
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: ben england <bengland@redhat.com>

4 files changed, 67 insertions, 26 deletions

diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index 69db8b70b59..96242dff77f 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1883,8 +1883,7 @@ rpcsvc_init_options (rpcsvc_t *svc, dict_t *options)
         if (!svc->register_portmap)
                 gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Portmap registration "
                         "disabled");
-
-        ret = rpcsvc_set_outstanding_rpc_limit (svc, options);
+        ret = 0;
 out:
         return ret;
 }
@@ -2016,10 +2015,15 @@ out:
 }
 
 /*
- * Reconfigure() the rpc.outstanding-rpc-limit param.
+ * Configure() the rpc.outstanding-rpc-limit param.
+ * If dict_get_int32() for dict-key "rpc.outstanding-rpc-limit" FAILS,
+ * it would set the value as "defvalue". Otherwise it would fetch the
+ * value and round up to multiple-of-8. defvalue must be +ve.
+ *
+ * NB: defval or set-value "0" is special which means unlimited/65536.
  */
 int
-rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options)
+rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options, int defvalue)
 {
         int            ret        = -1; /* FAILURE */
         int            rpclim     = 0;
@@ -2028,30 +2032,30 @@ rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options)
         if ((!svc) || (!options))
                 return (-1);
 
-        /* Reconfigure() the rpc.outstanding-rpc-limit param */
+        if ((defvalue < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) ||
+            (defvalue > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT)) {
+                return (-1);
+        }
+
+        /* Fetch the rpc.outstanding-rpc-limit from dict. */
         ret = dict_get_int32 (options, rpclimkey, &rpclim);
         if (ret < 0) {
                 /* Fall back to default for FAILURE */
-                rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
-        } else {
-                /* SUCCESS: round off to multiple of 8.
-                 * If the input value fails Boundary check, fall back to
-                 * default i.e. RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT.
-                 * NB: value 0 is special, means its unset i.e. unlimited.
-                 */
-                rpclim = ((rpclim + 8 - 1) >> 3) * 8;
-                if (rpclim < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) {
-                        rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
-                } else if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) {
-                        rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT;
-                }
+                rpclim = defvalue;
+        }
+
+        /* Round up to multiple-of-8. It must not exceed
+         * RPCSVC_MAX_OUTSTANDING_RPC_LIMIT.
+         */
+        rpclim = ((rpclim + 8 - 1) >> 3) * 8;
+        if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) {
+                rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT;
         }
 
         if (svc->outstanding_rpc_limit != rpclim) {
                 svc->outstanding_rpc_limit = rpclim;
                 gf_log (GF_RPCSVC, GF_LOG_INFO,
-                                   "Configured %s with value %d",
-                                   rpclimkey, rpclim);
+                        "Configured %s with value %d", rpclimkey, rpclim);
         }
 
         return (0);
@@ -2066,7 +2070,7 @@ rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options,
         rpcsvc_t          *svc              = NULL;
         int                ret              = -1;
 
-        if ((!ctx) || (!options))
+        if ((!xl) || (!ctx) || (!options))
                 return NULL;
 
         svc = GF_CALLOC (1, sizeof (*svc), gf_common_mt_rpcsvc_t);
diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h
index 30a969b11c0..5a6f930d359 100644
--- a/rpc/rpc-lib/src/rpcsvc.h
+++ b/rpc/rpc-lib/src/rpcsvc.h
@@ -38,7 +38,8 @@
 #define MAX_IOVEC 16
 #endif
 
-#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64
+#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64 /* Default for protocol/server */
+#define RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT 16 /* Default for nfs/server */
 #define RPCSVC_MAX_OUTSTANDING_RPC_LIMIT 65536
 #define RPCSVC_MIN_OUTSTANDING_RPC_LIMIT 0 /* No limit i.e. Unlimited */
 
@@ -597,7 +598,7 @@ rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options);
 int
 rpcsvc_set_root_squash (rpcsvc_t *svc, dict_t *options);
 int
-rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options);
+rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options, int defvalue);
 int
 rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen);
 rpcsvc_vector_sizer
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index 4ab5cbc9077..04cf030dc12 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -967,6 +967,15 @@ nfs_init_state (xlator_t *this)
                 goto free_foppool;
         }
 
+        ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc,
+                                  this->options,
+                                  RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
+        if (ret < 0) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "Failed to configure outstanding-rpc-limit");
+                goto free_foppool;
+        }
+
         nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc);
 
         this->private = (void *)nfs;
@@ -1210,6 +1219,17 @@ reconfigure (xlator_t *this, dict_t *options)
                         "rpcsvc reconfigure options failed");
                 return (-1);
         }
+
+        /* Reconfigure rpc.outstanding-rpc-limit */
+        ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc,
+                                       options,
+                                       RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
+        if (ret < 0) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "Failed to reconfigure outstanding-rpc-limit");
+                return (-1);
+        }
+
         regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc);
         if (nfs->register_portmap != regpmap) {
                 nfs->register_portmap = regpmap;
@@ -1741,7 +1761,7 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_INT,
           .min  = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT,
           .max  = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT,
-          .default_value = TOSTRING(RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT),
+          .default_value = TOSTRING(RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT),
           .description = "Parameter to throttle the number of incoming RPC "
                          "requests from a client. 0 means no limit (can "
                          "potentially run out of memory)"
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 56b83cb9a90..378aa393a50 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -746,7 +746,15 @@ reconfigure (xlator_t *this, dict_t *options)
 
         (void) rpcsvc_set_allow_insecure (rpc_conf, options);
         (void) rpcsvc_set_root_squash (rpc_conf, options);
-        (void) rpcsvc_set_outstanding_rpc_limit (rpc_conf, options);
+
+        ret = rpcsvc_set_outstanding_rpc_limit (rpc_conf, options,
+                                         RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to reconfigure outstanding-rpc-limit");
+                goto out;
+        }
+
         list_for_each_entry (listeners, &(rpc_conf->listeners), list) {
                 if (listeners->trans != NULL) {
                         if (listeners->trans->reconfigure )
@@ -860,12 +868,20 @@ init (xlator_t *this)
         /* RPC related */
         conf->rpc = rpcsvc_init (this, this->ctx, this->options, 0);
         if (conf->rpc == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_ERROR,
                         "creation of rpcsvc failed");
                 ret = -1;
                 goto out;
         }
 
+        ret = rpcsvc_set_outstanding_rpc_limit (conf->rpc, this->options,
+                                         RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to configure outstanding-rpc-limit");
+                goto out;
+        }
+
         ret = rpcsvc_create_listeners (conf->rpc, this->options,
                                        this->name);
         if (ret < 1) {