rda, glusterd: Change the max of rda-cache-limit to INFINITY

Issue:
The max value of rda-cache-limit is 1GB before this patch.
When parallel-readdir is enabled, there will be many instances of
readdir-ahead, hence the rda-cache-limit depends on the number of
instances. Eg: On a volume with distribute count 4, rda-cache-limit
when parallel-readdir is enabled, will be 4GB instead of 1GB.
Consider a followinf sequence of operations:
- Enable parallel readdir
- Set rda-cache-limit to lets say 3GB
- Disable parallel-readdir, this results in one instance of readdir-ahead
  and the rda-cache-limit will be back to 1GB, but the current value is 3GB
  and hence the mount will stop working as 3GB > max 1GB.

Solution:
To fix this, we can limit the cache to 1GB even when parallel-readdir
is enabled. But there is no necessity to limit the cache to 1GB, it
can be increased if the system has enough resources. Hence getting rid
of the rda-cache-limit max value is more apt. If we just change the
rda-cache-limit max to INFINITY, we will render older(<3.11) clients
broken, when the rda-cache-limit is set to > 1GB (as the older clients
still expect a value < 1GB). To safely change the max value of
rda-cache-limit to INFINITY, add a check in glusted to verify all
the clients are > 3.11 if the value exceeds 1GB.

Change-Id: Id0cdda3b053287b659c7bf511b13db2e45b92032
BUG: 1446516
Signed-off-by: Poornima G <pgurusid@redhat.com>
Reviewed-on: https://review.gluster.org/17338
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

4 files changed, 55 insertions, 2 deletions

diff --git a/tests/bugs/readdir-ahead/bug-1439640.t b/tests/bugs/readdir-ahead/bug-1439640.t
index cfa5d1ff659..cc6c829fccc 100755
--- a/tests/bugs/readdir-ahead/bug-1439640.t
+++ b/tests/bugs/readdir-ahead/bug-1439640.t
@@ -27,4 +27,4 @@ TEST ! $CLI volume set $V0 rda-request-size 87adh
 TEST $CLI volume set $V0 rda-cache-limit 10MB
 TEST $CLI volume set $V0 rda-request-size 128KB
 
-#cleanup;
+cleanup;
diff --git a/tests/bugs/readdir-ahead/bug-1446516.t b/tests/bugs/readdir-ahead/bug-1446516.t
new file mode 100755
index 00000000000..2bf6520d861
--- /dev/null
+++ b/tests/bugs/readdir-ahead/bug-1446516.t
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+cleanup;
+
+TEST glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1..4}
+TEST $CLI volume start $V0
+
+TEST $CLI volume set $V0 parallel-readdir on
+
+TEST $CLI volume set $V0 rda-cache-limit 4GB
+
+TEST $CLI volume set $V0 parallel-readdir off
+
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 2b7abf1b7ff..caa83b5a184 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1004,6 +1004,36 @@ out:
 
 
 static int
+validate_rda_cache_limit (glusterd_volinfo_t *volinfo, dict_t *dict,
+                          char *key, char *value, char **op_errstr)
+{
+        int          ret             = 0;
+        uint64_t     rda_cache_size  = 0;
+
+        ret = gf_string2bytesize_uint64 (value, &rda_cache_size);
+        if (ret < 0)
+                goto out;
+
+        if (rda_cache_size <= (1 * GF_UNIT_GB))
+                goto out;
+
+        /* With release 3.11 the max value of rda_cache_limit is changed from
+         * 1GB to INFINITY. If there are clients older than 3.11 and the value
+         * of rda-cache-limit is set to > 1GB, the older clients will stop
+         * working. Hence if a user is setting rda-cache-limit to > 1GB
+         * ensure that all the clients are 3.11 or greater.
+         */
+        ret = glusterd_check_client_op_version_support (volinfo->volname,
+                                                        GD_OP_VERSION_3_11_0,
+                                                        op_errstr);
+out:
+        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+
+        return ret;
+}
+
+
+static int
 validate_worm_period (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
                char *value, char **op_errstr)
 {
@@ -3286,6 +3316,7 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = DOC,
           .flags       = OPT_FLAG_CLIENT_OPT,
           .op_version  = GD_OP_VERSION_3_9_1,
+          .validate_fn = validate_rda_cache_limit
         },
         { .key         = "performance.nl-cache-positive-entry",
           .voltype     = "performance/nl-cache",
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.c b/xlators/performance/readdir-ahead/src/readdir-ahead.c
index 2ffcf25d8a7..4e7f2b9a879 100644
--- a/xlators/performance/readdir-ahead/src/readdir-ahead.c
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.c
@@ -23,6 +23,7 @@
  * preloads on the directory.
  */
 
+#include <math.h>
 #include "glusterfs.h"
 #include "xlator.h"
 #include "call-stub.h"
@@ -739,7 +740,7 @@ struct volume_options options[] = {
         { .key = {"rda-cache-limit"},
           .type = GF_OPTION_TYPE_SIZET,
           .min = 0,
-          .max = 1 * GF_UNIT_GB,
+          .max = INFINITY,
           .default_value = "10MB",
           .description = "maximum size of cache consumed by readdir-ahead "
                          "xlator. This value is global and total memory "