glusterd: Introduce daemon-log-level cluster wide option

This option, applicable to the node level daemons can be very helpful in
controlling the log level of these services. Please note any daemon
which is started prior to setting the specific value of this option (if
not INFO) will need to go through a restart to have this change into
effect.

Change-Id: I7f6d2620bab2b094c737f5cc816bc093e9c9c4c9
fixes: bz#1597473
Signed-off-by: Atin Mukherjee <amukherj@redhat.com>

7 files changed, 162 insertions, 1 deletions

diff --git a/tests/bugs/glusterd/daemon-log-level-option.t b/tests/bugs/glusterd/daemon-log-level-option.t
new file mode 100644
index 00000000000..66e55e3d758
--- /dev/null
+++ b/tests/bugs/glusterd/daemon-log-level-option.t
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+
+function Info_messages_count() {
+        local shd_log=$1
+        cat $shd_log | grep " I " | wc -l
+}
+
+function Warning_messages_count() {
+        local shd_log=$1
+        cat $shd_log | grep " W " | wc -l
+}
+
+function Debug_messages_count() {
+        local shd_log=$1
+        cat $shd_log | grep " D " | wc -l
+}
+
+function Trace_messages_count() {
+        local shd_log=$1
+        cat $shd_log | grep " T " | wc -l
+}
+
+cleanup;
+
+# Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+# set cluster.daemon-log-level option to DEBUG
+TEST $CLI volume set all cluster.daemon-log-level DEBUG
+
+#Create a 3X2 distributed-replicate volume
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1..6};
+TEST $CLI volume start $V0
+
+# log should not have any trace messages
+EXPECT 0 Trace_messages_count "/var/log/glusterfs/glustershd.log"
+
+# stop the volume and remove glustershd log
+TEST $CLI volume stop $V0
+rm -f /var/log/glusterfs/glustershd.log
+
+# set cluster.daemon-log-level option to INFO and start the volume
+TEST $CLI volume set all cluster.daemon-log-level INFO
+TEST $CLI volume start $V0
+
+# log should not have any debug messages
+EXPECT 0 Debug_messages_count "/var/log/glusterfs/glustershd.log"
+
+# log should not have any trace messages
+EXPECT 0 Trace_messages_count "/var/log/glusterfs/glustershd.log"
+
+# stop the volume and remove glustershd log
+TEST $CLI volume stop $V0
+rm -f /var/log/glusterfs/glustershd.log
+
+# set cluster.daemon-log-level option to WARNING and start the volume
+TEST $CLI volume set all cluster.daemon-log-level WARNING
+TEST $CLI volume start $V0
+
+# log should not have any info messages
+EXPECT 0 Info_messages_count "/var/log/glusterfs/glustershd.log"
+
+# log should not have any debug messages
+EXPECT 0 Debug_messages_count "/var/log/glusterfs/glustershd.log"
+
+# log should not have any trace messages
+EXPECT 0 Trace_messages_count "/var/log/glusterfs/glustershd.log"
+
+# stop the volume and remove glustershd log
+TEST $CLI volume stop $V0
+rm -f /var/log/glusterfs/glustershd.log
+
+# set cluster.daemon-log-level option to ERROR and start the volume
+TEST $CLI volume set all cluster.daemon-log-level ERROR
+TEST $CLI volume start $V0
+
+# log should not have any info messages
+EXPECT 0 Info_messages_count "/var/log/glusterfs/glustershd.log"
+
+# log should not have any warning messages
+EXPECT 0 Warning_messages_count "/var/log/glusterfs/glustershd.log"
+
+# log should not have any debug messages
+EXPECT 0 Debug_messages_count "/var/log/glusterfs/glustershd.log"
+
+# log should not have any trace messages
+EXPECT 0 Trace_messages_count "/var/log/glusterfs/glustershd.log"
+
+cleanup
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 64ff9ffcaf4..6d513bb90b1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -4618,6 +4618,7 @@ gd_is_global_option (char *opt_key)
                 strcmp (opt_key, GLUSTERD_GLOBAL_OP_VERSION_KEY) == 0 ||
                 strcmp (opt_key, GLUSTERD_BRICK_MULTIPLEX_KEY) == 0 ||
                 strcmp (opt_key, GLUSTERD_LOCALTIME_LOGGING_KEY) == 0 ||
+                strcmp (opt_key, GLUSTERD_DAEMON_LOG_LEVEL_KEY) == 0 ||
                 strcmp (opt_key, GLUSTERD_MAX_OP_VERSION_KEY) == 0);
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index 218071396d9..29b97570630 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -636,7 +636,8 @@ GLFS_MSGID(GLUSTERD,
         GD_MSG_LOCALTIME_LOGGING_DISABLE,
         GD_MSG_PORTS_EXHAUSTED,
         GD_MSG_CHANGELOG_GET_FAIL,
-        GD_MSG_MANAGER_FUNCTION_FAILED
+        GD_MSG_MANAGER_FUNCTION_FAILED,
+        GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL
 );
 
 #endif /* !_GLUSTERD_MESSAGES_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index c1e7612c377..e49b9c720e5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -87,6 +87,7 @@ glusterd_all_vol_opts valid_all_vol_opts[] = {
          * dynamic value depending on the memory specifications per node */
         { GLUSTERD_BRICKMUX_LIMIT_KEY,          "0"},
         { GLUSTERD_LOCALTIME_LOGGING_KEY,       "disable"},
+        { GLUSTERD_DAEMON_LOG_LEVEL_KEY,        "INFO"},
         { NULL },
 };
 
@@ -937,6 +938,47 @@ out:
 }
 
 static int
+glusterd_validate_daemon_log_level (char *key, char *value, char *errstr)
+{
+        int32_t            ret                      = -1;
+        xlator_t          *this                     = NULL;
+        glusterd_conf_t   *conf                     = NULL;
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        GF_VALIDATE_OR_GOTO (this->name, key, out);
+        GF_VALIDATE_OR_GOTO (this->name, value, out);
+        GF_VALIDATE_OR_GOTO (this->name, errstr, out);
+
+        ret = 0;
+
+        if (strcmp (key, GLUSTERD_DAEMON_LOG_LEVEL_KEY)) {
+                goto out;
+        }
+
+        if ((strcmp (value, "INFO")) &&
+            (strcmp (value, "WARNING")) &&
+            (strcmp (value, "DEBUG")) &&
+            (strcmp (value, "TRACE")) &&
+            (strcmp (value, "ERROR"))) {
+                snprintf (errstr, PATH_MAX,
+                          "Invalid option(%s). Valid options "
+                          "are 'INFO' or 'WARNING' or 'ERROR' or 'DEBUG' or "
+                          " 'TRACE'", value);
+                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
+                        GD_MSG_INVALID_ENTRY, "%s", errstr);
+                ret = -1;
+        }
+
+out:
+        return ret;
+}
+
+static int
 glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
 {
         int                   ret                         = -1;
@@ -1333,6 +1375,15 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
+                ret = glusterd_validate_daemon_log_level (key, value, errstr);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL,
+                                "Failed to validate daemon-log-level volume "
+                                "options");
+                        goto out;
+                }
+
                 if (volinfo) {
                         ret = glusterd_volinfo_get (volinfo,
                                                 VKEY_FEATURES_TRASH, &val_dup);
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
index ba948b4c6f6..ebb288c7191 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
@@ -151,6 +151,8 @@ glusterd_svc_start (glusterd_svc_t *svc, int flags, dict_t *cmdline)
         xlator_t            *this                       = NULL;
         char                 valgrind_logfile[PATH_MAX] = {0};
         char                *localtime_logging          = NULL;
+        char                *log_level                  = NULL;
+        char                 daemon_log_level[30]       = {0};
 
         this = THIS;
         GF_ASSERT (this);
@@ -196,6 +198,12 @@ glusterd_svc_start (glusterd_svc_t *svc, int flags, dict_t *cmdline)
                 if (strcmp (localtime_logging, "enable") == 0)
                         runner_add_arg (&runner, "--localtime-logging");
         }
+        if (dict_get_str (priv->opts, GLUSTERD_DAEMON_LOG_LEVEL_KEY,
+                          &log_level) == 0) {
+                snprintf (daemon_log_level, 30, "--log-level=%s", log_level);
+                runner_add_arg (&runner, daemon_log_level);
+        }
+
         if (cmdline)
                 dict_foreach (cmdline, svc_add_args, (void *) &runner);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 5af1b3eaff9..26a076905e9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -3779,6 +3779,12 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .op_version  = GD_OP_VERSION_3_12_0,
           .validate_fn = validate_boolean
         },
+        { .key         = GLUSTERD_DAEMON_LOG_LEVEL_KEY,
+          .voltype     = "mgmt/glusterd",
+          .type        = GLOBAL_NO_DOC,
+          .value       = "INFO",
+          .op_version  = GD_OP_VERSION_4_2_0
+        },
         { .key        = "debug.delay-gen",
           .voltype    = "debug/delay-gen",
           .option     = "!debug",
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index e7a4c80a635..33413976e6d 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -56,6 +56,7 @@
 #define GLUSTERD_BRICK_MULTIPLEX_KEY    "cluster.brick-multiplex"
 #define GLUSTERD_BRICKMUX_LIMIT_KEY     "cluster.max-bricks-per-process"
 #define GLUSTERD_LOCALTIME_LOGGING_KEY  "cluster.localtime-logging"
+#define GLUSTERD_DAEMON_LOG_LEVEL_KEY   "cluster.daemon-log-level"
 
 #define GLUSTERD_SNAPS_MAX_HARD_LIMIT 256
 #define GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT 90