fuse: make background queue length configurable

* also make 'congestion_threshold' an option
* make 'congestion_threshold' as 75% of background queue length if
  not explicitely specified
* in glusterfsd.c, moved all the fuse option dictionary setting
  code to separate function

Change-Id: Ie1680eefaed9377720770a09222282321bd4132e
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 845214
Reviewed-on: http://review.gluster.org/3830
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

6 files changed, 166 insertions, 57 deletions

diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 262ef15b69e..fd3fddfb72c 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -186,6 +186,12 @@ static struct argp_option gf_options[] = {
 	{"gid-timeout", ARGP_GID_TIMEOUT_KEY, "SECONDS", 0,
 	 "Set auxilary group list timeout to SECONDS for fuse translator "
 	 "[default: 0]"},
+	{"background-qlen", ARGP_FUSE_BACKGROUND_QLEN_KEY, "N", 0,
+	 "Set fuse module's background queue length to N "
+	 "[default: 64]"},
+	{"congestion-threshold", ARGP_FUSE_CONGESTION_THRESHOLD_KEY, "N", 0,
+	 "Set fuse module's congestion threshold to N "
+	 "[default: 48]"},
         {"client-pid", ARGP_CLIENT_PID_KEY, "PID", OPTION_HIDDEN,
          "client will authenticate itself with process id PID to server"},
         {"user-map-root", ARGP_USER_MAP_ROOT_KEY, "USER", OPTION_HIDDEN,
@@ -209,50 +215,17 @@ int glusterfs_mgmt_init (glusterfs_ctx_t *ctx);
 int glusterfs_listener_init (glusterfs_ctx_t *ctx);
 int glusterfs_listener_stop (glusterfs_ctx_t *ctx);
 
-int
-create_fuse_mount (glusterfs_ctx_t *ctx)
+
+static int
+set_fuse_mount_options (glusterfs_ctx_t *ctx, dict_t *options)
 {
         int              ret = 0;
         cmd_args_t      *cmd_args = NULL;
-        xlator_t        *master = NULL;
         char            *mount_point = NULL;
         char            cwd[PATH_MAX] = {0,};
 
         cmd_args = &ctx->cmd_args;
 
-        if (!cmd_args->mount_point) {
-                gf_log ("", GF_LOG_TRACE,
-                        "mount point not found, not a client process");
-                return 0;
-        }
-
-        if (ctx->process_mode != GF_CLIENT_PROCESS) {
-                gf_log("glusterfsd", GF_LOG_ERROR,
-                       "Not a client process, not performing mount operation");
-                return -1;
-        }
-
-        master = GF_CALLOC (1, sizeof (*master),
-                            gfd_mt_xlator_t);
-        if (!master)
-                goto err;
-
-        master->name = gf_strdup ("fuse");
-        if (!master->name)
-                goto err;
-
-        if (xlator_set_type (master, "mount/fuse") == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "MOUNT-POINT %s initialization failed",
-                        cmd_args->mount_point);
-                goto err;
-        }
-
-        master->ctx      = ctx;
-        master->options  = get_new_dict ();
-        if (!master->options)
-                goto err;
-
         /* Check if mount-point is absolute path,
          * if not convert to absolute path by concating with CWD
          */
@@ -274,7 +247,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         } else
                 mount_point = gf_strdup (cmd_args->mount_point);
 
-        ret = dict_set_dynstr (master->options, ZR_MOUNTPOINT_OPT, mount_point);
+        ret = dict_set_dynstr (options, ZR_MOUNTPOINT_OPT, mount_point);
         if (ret < 0) {
                 gf_log ("glusterfsd", GF_LOG_ERROR,
                         "failed to set mount-point to options dictionary");
@@ -282,7 +255,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->fuse_attribute_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_ATTR_TIMEOUT_OPT,
+                ret = dict_set_double (options, ZR_ATTR_TIMEOUT_OPT,
                                        cmd_args->fuse_attribute_timeout);
 
                 if (ret < 0) {
@@ -294,7 +267,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->fuse_entry_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_ENTRY_TIMEOUT_OPT,
+                ret = dict_set_double (options, ZR_ENTRY_TIMEOUT_OPT,
                                        cmd_args->fuse_entry_timeout);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -305,7 +278,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->fuse_negative_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_NEGATIVE_TIMEOUT_OPT,
+                ret = dict_set_double (options, ZR_NEGATIVE_TIMEOUT_OPT,
                                        cmd_args->fuse_negative_timeout);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -316,7 +289,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->client_pid_set) {
-                ret = dict_set_int32 (master->options, "client-pid",
+                ret = dict_set_int32 (options, "client-pid",
                                       cmd_args->client_pid);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -327,7 +300,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->uid_map_root) {
-                ret = dict_set_int32 (master->options, "uid-map-root",
+                ret = dict_set_int32 (options, "uid-map-root",
                                       cmd_args->uid_map_root);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -338,7 +311,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->volfile_check) {
-                ret = dict_set_int32 (master->options, ZR_STRICT_VOLFILE_CHECK,
+                ret = dict_set_int32 (options, ZR_STRICT_VOLFILE_CHECK,
                                       cmd_args->volfile_check);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -349,7 +322,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->dump_fuse) {
-                ret = dict_set_static_ptr (master->options, ZR_DUMP_FUSE,
+                ret = dict_set_static_ptr (options, ZR_DUMP_FUSE,
                                            cmd_args->dump_fuse);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -360,7 +333,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->acl) {
-                ret = dict_set_static_ptr (master->options, "acl", "on");
+                ret = dict_set_static_ptr (options, "acl", "on");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key acl");
@@ -369,7 +342,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->selinux) {
-                ret = dict_set_static_ptr (master->options, "selinux", "on");
+                ret = dict_set_static_ptr (options, "selinux", "on");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key selinux");
@@ -378,7 +351,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->read_only) {
-                ret = dict_set_static_ptr (master->options, "read-only", "on");
+                ret = dict_set_static_ptr (options, "read-only", "on");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key read-only");
@@ -387,7 +360,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
 	if (cmd_args->fopen_keep_cache) {
-		ret = dict_set_static_ptr(master->options, "fopen-keep-cache",
+		ret = dict_set_static_ptr(options, "fopen-keep-cache",
 			"on");
 		if (ret < 0) {
 			gf_log("glusterfsd", GF_LOG_ERROR,
@@ -398,7 +371,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
 	}
 
 	if (cmd_args->gid_timeout) {
-		ret = dict_set_int32(master->options, "gid-timeout",
+		ret = dict_set_int32(options, "gid-timeout",
 			cmd_args->gid_timeout);
 		if (ret < 0) {
 			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
@@ -406,10 +379,28 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
 			goto err;
 		}
 	}
+	if (cmd_args->background_qlen) {
+		ret = dict_set_int32 (options, "background-qlen",
+                                      cmd_args->background_qlen);
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
+                               "value for key background-qlen");
+			goto err;
+		}
+	}
+	if (cmd_args->congestion_threshold) {
+		ret = dict_set_int32 (options, "congestion-threshold",
+                                      cmd_args->congestion_threshold);
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
+                               "value for key congestion-threshold");
+			goto err;
+		}
+	}
 
         switch (cmd_args->fuse_direct_io_mode) {
         case GF_OPTION_DISABLE: /* disable */
-                ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT,
+                ret = dict_set_static_ptr (options, ZR_DIRECT_IO_OPT,
                                            "disable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -419,7 +410,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
                 }
                 break;
         case GF_OPTION_ENABLE: /* enable */
-                ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT,
+                ret = dict_set_static_ptr (options, ZR_DIRECT_IO_OPT,
                                            "enable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -436,14 +427,64 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (!cmd_args->no_daemon_mode) {
-                 ret = dict_set_static_ptr (master->options, "sync-to-mount",
-                                            "enable");
+                ret = dict_set_static_ptr (options, "sync-to-mount",
+                                           "enable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key sync-mtab");
                         goto err;
                 }
         }
+        ret = 0;
+err:
+        return ret;
+}
+
+int
+create_fuse_mount (glusterfs_ctx_t *ctx)
+{
+        int              ret = 0;
+        cmd_args_t      *cmd_args = NULL;
+        xlator_t        *master = NULL;
+
+        cmd_args = &ctx->cmd_args;
+
+        if (!cmd_args->mount_point) {
+                gf_log ("", GF_LOG_TRACE,
+                        "mount point not found, not a client process");
+                return 0;
+        }
+
+        if (ctx->process_mode != GF_CLIENT_PROCESS) {
+                gf_log("glusterfsd", GF_LOG_ERROR,
+                       "Not a client process, not performing mount operation");
+                return -1;
+        }
+
+        master = GF_CALLOC (1, sizeof (*master),
+                            gfd_mt_xlator_t);
+        if (!master)
+                goto err;
+
+        master->name = gf_strdup ("fuse");
+        if (!master->name)
+                goto err;
+
+        if (xlator_set_type (master, "mount/fuse") == -1) {
+                gf_log ("glusterfsd", GF_LOG_ERROR,
+                        "MOUNT-POINT %s initialization failed",
+                        cmd_args->mount_point);
+                goto err;
+        }
+
+        master->ctx      = ctx;
+        master->options  = get_new_dict ();
+        if (!master->options)
+                goto err;
+
+        ret = set_fuse_mount_options (ctx, master->options);
+        if (ret)
+                goto err;
 
         ret = xlator_init (master);
         if (ret) {
@@ -870,6 +911,20 @@ parse_opts (int key, char *arg, struct argp_state *state)
 
 		argp_failure(state, -1, 0, "unknown group list timeout %s", arg);
 		break;
+        case ARGP_FUSE_BACKGROUND_QLEN_KEY:
+                if (!gf_string2int (arg, &cmd_args->background_qlen))
+                        break;
+
+                argp_failure (state, -1, 0,
+                              "unknown background qlen option %s", arg);
+                break;
+        case ARGP_FUSE_CONGESTION_THRESHOLD_KEY:
+                if (!gf_string2int (arg, &cmd_args->congestion_threshold))
+                        break;
+
+                argp_failure (state, -1, 0,
+                              "unknown congestion threshold option %s", arg);
+                break;
 	}
 
         return 0;
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index 4f08cd31a60..05851160385 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -90,6 +90,8 @@ enum argp_option_keys {
         ARGP_SELINUX_KEY                  = 158,
 	ARGP_FOPEN_KEEP_CACHE_KEY	  = 159,
 	ARGP_GID_TIMEOUT_KEY		  = 160,
+	ARGP_FUSE_BACKGROUND_QLEN_KEY     = 161,
+	ARGP_FUSE_CONGESTION_THRESHOLD_KEY = 162,
 };
 
 struct _gfd_vol_top_priv_t {
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 8fc49d3d2c5..622651b4c9e 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -313,7 +313,8 @@ struct _cmd_args {
         pid_t            client_pid;
         int              client_pid_set;
         unsigned         uid_map_root;
-
+        int              background_qlen;
+        int              congestion_threshold;
 
 	/* key args */
 	char            *mount_point;
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index af6de8cd07c..7915964ed95 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -3498,9 +3498,8 @@ fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
 	}
 
         if (fini->minor >= 13) {
-                /* these values seemed to work fine during testing */
-                fino.max_background = 64;
-                fino.congestion_threshold = 48;
+                fino.max_background = priv->background_qlen;
+                fino.congestion_threshold = priv->congestion_threshold;
         }
         if (fini->minor < 9)
                 *priv->msg0_len_p = sizeof(*finh) + FUSE_COMPAT_WRITE_IN_SIZE;
@@ -4650,6 +4649,32 @@ init (xlator_t *this_xl)
 		goto cleanup_exit;
 	}
 
+        /* default values seemed to work fine during testing */
+	GF_OPTION_INIT ("background-qlen", priv->background_qlen, int32,
+                        cleanup_exit);
+	GF_OPTION_INIT ("congestion-threshold", priv->congestion_threshold,
+                        int32, cleanup_exit);
+
+        /* user has set only background-qlen, not congestion-threshold,
+           use the fuse kernel driver formula to set congestion. ie, 75% */
+        if (dict_get (this_xl->options, "background-qlen") &&
+            !dict_get (this_xl->options, "congestion-threshold")) {
+                priv->congestion_threshold = (priv->background_qlen * 3) / 4;
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control as 75%% of "
+                        "background-queue length (ie, (.75 * %d) = %d",
+                        priv->background_qlen, priv->congestion_threshold);
+        }
+
+        /* congestion should not be higher than background queue length */
+        if (priv->congestion_threshold > priv->background_qlen) {
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control same as "
+                        "background-queue length (%d)",
+                        priv->background_qlen);
+                priv->congestion_threshold = priv->background_qlen;
+        }
+
         cmd_args = &this_xl->ctx->cmd_args;
         fsname = cmd_args->volfile;
         if (!fsname && cmd_args->volfile_server) {
@@ -4833,5 +4858,17 @@ struct volume_options options[] = {
 	  .type = GF_OPTION_TYPE_BOOL,
 	  .default_value = "false"
 	},
+        { .key  = {"background-qlen"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "64",
+          .min = 16,
+          .max = (64 * GF_UNIT_KB),
+        },
+        { .key  = {"congestion-threshold"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "48",
+          .min = 12,
+          .max = (64 * GF_UNIT_KB),
+        },
         { .key = {NULL} },
 };
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index 055cbd3110c..9b90d0a47fd 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -123,6 +123,10 @@ struct fuse_private {
 
         /* For communicating with separate mount thread. */
         int                  status_pipe[2];
+
+        /* for fuse queue length and congestion threshold */
+        int background_qlen;
+        int congestion_threshold;
 };
 typedef struct fuse_private fuse_private_t;
 
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index a0a31660ab3..3bbbc3a06e9 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -144,6 +144,14 @@ start_glusterfs ()
 	cmd_line=$(echo "$cmd_line --fopen-keep-cache");
     fi
 
+    if [ -n "$bg_qlen" ]; then
+	cmd_line=$(echo "$cmd_line --background-qlen=$bg_qlen");
+    fi
+
+    if [ -n "$cong_threshold" ]; then
+	cmd_line=$(echo "$cmd_line --congestion-threshold=$cong_threshold");
+    fi
+
     # for rdma volume, we have to fetch volfile with '.rdma' added
     # to volume name, so that it fetches the right client vol file
     volume_id_rdma="";
@@ -335,6 +343,8 @@ main ()
                             "entry-timeout")    entry_timeout=$value ;;
                             "negative-timeout")    negative_timeout=$value ;;
 			    "gid-timeout")	gid_timeout=$value ;;
+			    "background-qlen")	bg_qlen=$value ;;
+			    "congestion-threshold")	cong_threshold=$value ;;
                             *) echo "unknown option $key (ignored)" ;;
                         esac
                 esac