From d2de3f66393d4f737e21eedab4200563a60f8bcc Mon Sep 17 00:00:00 2001 From: Mohit Agrawal Date: Mon, 15 Apr 2019 10:34:34 +0530 Subject: core: Log level changes do not effect on running client process Problem: commit c34e4161f3cb6539ec83a9020f3d27eb4759a975 set log-level per xlator during reconfigure only for a brick process not for the client process. Solution: 1) Change per xlator log-level only if brick_mux is enabled.To make sure about brick multiplex introudce a flag brick_mux at ctx->cmd_args. Note: There are two other changes done with this patch 1) Ignore client-log-level option to attach a brick with already running brick if brick_mux is enabled 2) Add a log to print pid of the running process to make easier debugging > Change-Id: I39e85de778e150d0685cd9a79425ce8b4783f9c9 > Signed-off-by: Mohit Agrawal > Fixes: bz#1696046 > (Cherry picked from commit 798aadbe51a9a02dd98a0f861cc239ecf7c8ed57) > (Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/22495/) Change-Id: If91682830f894ab8f6857f19dcb1797fc15ca64c Fixes: bz#1699715 Signed-off-by: Mohit Agrawal --- glusterfsd/src/glusterfsd-messages.h | 3 +- glusterfsd/src/glusterfsd.c | 17 ++++- glusterfsd/src/glusterfsd.h | 1 + libglusterfs/src/glusterfs/glusterfs.h | 1 + tests/bugs/glusterd/bug-1696046.t | 113 +++++++++++++++++++++++++++++ xlators/debug/io-stats/src/io-stats.c | 22 +++--- xlators/mgmt/glusterd/src/glusterd-utils.c | 7 ++ 7 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 tests/bugs/glusterd/bug-1696046.t diff --git a/glusterfsd/src/glusterfsd-messages.h b/glusterfsd/src/glusterfsd-messages.h index 602cd9ecfd5..ce6c8ca56a5 100644 --- a/glusterfsd/src/glusterfsd-messages.h +++ b/glusterfsd/src/glusterfsd-messages.h @@ -35,6 +35,7 @@ GLFS_MSGID(GLUSTERFSD, glusterfsd_msg_1, glusterfsd_msg_2, glusterfsd_msg_3, glusterfsd_msg_28, glusterfsd_msg_29, glusterfsd_msg_30, glusterfsd_msg_31, glusterfsd_msg_32, glusterfsd_msg_33, glusterfsd_msg_34, glusterfsd_msg_35, glusterfsd_msg_36, - glusterfsd_msg_37, glusterfsd_msg_38, glusterfsd_msg_39); + glusterfsd_msg_37, glusterfsd_msg_38, glusterfsd_msg_39, + glusterfsd_msg_40); #endif /* !_GLUSTERFSD_MESSAGES_H_ */ diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index 5d46b3d9f45..fc86cbdccc9 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -275,6 +275,7 @@ static struct argp_option gf_options[] = { "attribute, dentry and page-cache. " "Disable this only if same files/directories are not accessed across " "two different mounts concurrently [default: \"on\"]"}, + {"brick-mux", ARGP_BRICK_MUX_KEY, 0, 0, "Enable brick mux. "}, {0, 0, 0, 0, "Miscellaneous Options:"}, { 0, @@ -711,7 +712,6 @@ create_fuse_mount(glusterfs_ctx_t *ctx) xlator_t *master = NULL; cmd_args = &ctx->cmd_args; - if (!cmd_args->mount_point) { gf_msg_trace("glusterfsd", 0, "mount point not found, not a client process"); @@ -1099,6 +1099,10 @@ parse_opts(int key, char *arg, struct argp_state *state) cmd_args->thin_client = _gf_true; break; + case ARGP_BRICK_MUX_KEY: + cmd_args->brick_mux = _gf_true; + break; + case ARGP_PID_FILE_KEY: cmd_args->pid_file = gf_strdup(arg); break; @@ -1216,7 +1220,6 @@ parse_opts(int key, char *arg, struct argp_state *state) case ARGP_KEY_ARG: if (state->arg_num >= 1) argp_usage(state); - cmd_args->mount_point = gf_strdup(arg); break; @@ -2549,6 +2552,8 @@ postfork: if (ret) goto out; } + gf_log("glusterfs", GF_LOG_INFO, "Pid of current running process is %d", + getpid()); ret = gf_log_inject_timer_event(ctx); glusterfs_signals_setup(ctx); @@ -2814,6 +2819,14 @@ main(int argc, char *argv[]) if (ret) goto out; + /* set brick_mux mode only for server process */ + if ((ctx->process_mode != GF_SERVER_PROCESS) && cmd->brick_mux) { + gf_msg("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_40, + "command line argument --brick-mux is valid only for brick " + "process"); + goto out; + } + /* log the version of glusterfs running here along with the actual command line options. */ { diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h index 35cf6d88b7a..fa55789c611 100644 --- a/glusterfsd/src/glusterfsd.h +++ b/glusterfsd/src/glusterfsd.h @@ -111,6 +111,7 @@ enum argp_option_keys { ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY = 189, ARGP_FUSE_LRU_LIMIT_KEY = 190, ARGP_FUSE_AUTO_INVAL_KEY = 191, + ARGP_BRICK_MUX_KEY = 192 }; struct _gfd_vol_top_priv { diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h index 7c6af090fd8..263213d4048 100644 --- a/libglusterfs/src/glusterfs/glusterfs.h +++ b/libglusterfs/src/glusterfs/glusterfs.h @@ -575,6 +575,7 @@ struct _cmd_args { int fuse_flush_handle_interrupt; int fuse_auto_inval; + bool brick_mux; }; typedef struct _cmd_args cmd_args_t; diff --git a/tests/bugs/glusterd/bug-1696046.t b/tests/bugs/glusterd/bug-1696046.t new file mode 100644 index 00000000000..e1c1eb2ceb9 --- /dev/null +++ b/tests/bugs/glusterd/bug-1696046.t @@ -0,0 +1,113 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +function count_up_bricks { + $CLI --xml volume status $1 | grep '1' | wc -l +} + +function count_brick_processes { + pgrep glusterfsd | wc -l +} + +logdir=`gluster --print-logdir` + +## Start and create a volume +TEST glusterd; +TEST pidof glusterd; + +TEST $CLI volume set all cluster.brick-multiplex on +TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{1,2,3}; +TEST $CLI volume create $V1 replica 3 $H0:$B0/${V1}{1,2,3}; + +## Start volume and verify +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; +TEST $CLI volume start $V1; +EXPECT 'Started' volinfo_field $V1 'Status'; + + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 4 count_up_bricks $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 4 count_up_bricks $V1 + +EXPECT 1 count_brick_processes + +# Mount V0 +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; + +function client-log-file-name() +{ + logfilename=$M0".log" + echo ${logfilename:1} | tr / - +} + +function brick-log-file-name() +{ + logfilename=$B0"/"$V0"1.log" + echo ${logfilename:1} | tr / - +} + +log_file=$logdir"/"`client-log-file-name` +nofdlog=$(cat $log_file | grep " D " | wc -l) +TEST [ $((nofdlog)) -eq 0 ] + +brick_log_file=$logdir"/bricks/"`brick-log-file-name` +nofdlog=$(cat $brick_log_file | grep " D " | wc -l) +TEST [ $((nofdlog)) -eq 0 ] + +## Set brick-log-level to DEBUG +TEST $CLI volume set $V0 diagnostics.brick-log-level DEBUG + +# Do some operation +touch $M0/file1 + +# Check debug message debug message should be exist only for V0 +# Server xlator is common in brick_mux so after enabling DEBUG log +# some debug message should be available for other xlators like posix + +brick_log_file=$logdir"/bricks/"`brick-log-file-name` +nofdlog=$(cat $brick_log_file | grep file1 | grep -v server | wc -l) +TEST [ $((nofdlog)) -ne 0 ] + +#Check if any debug log exist in client-log file +nofdlog=$(cat $log_file | grep " D " | wc -l) +TEST [ $((nofdlog)) -eq 0 ] + +## Set brick-log-level to INFO +TEST $CLI volume set $V0 diagnostics.brick-log-level INFO + +## Set client-log-level to DEBUG +TEST $CLI volume set $V0 diagnostics.client-log-level DEBUG + +# Do some operation +touch $M0/file2 + +nofdlog=$(cat $brick_log_file | grep " D " | grep file2 | wc -l) +TEST [ $((nofdlog)) -eq 0 ] + +nofdlog=$(cat $log_file | grep " D " | wc -l) +TEST [ $((nofdlog)) -ne 0 ] + +# Unmount V0 +TEST umount $M0 + +#Mount V1 +TEST glusterfs --volfile-id=$V1 --volfile-server=$H0 --entry-timeout=0 $M0; + +#do some operation +touch $M0/file3 + + +# DEBUG log level is enabled only for V0 so no debug message should be available +# in log specific to file2 creation except for server xlator, server xlator is +# common xlator in brick mulitplex +nofdlog=$(cat $brick_log_file | grep file3 | grep -v server | wc -l) +TEST [ $((nofdlog)) -eq 0 ] + +# Unmount V1 +TEST umount $M0 + +cleanup; diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c index f12191fb8df..13545bdaa3b 100644 --- a/xlators/debug/io-stats/src/io-stats.c +++ b/xlators/debug/io-stats/src/io-stats.c @@ -3704,19 +3704,23 @@ xlator_set_loglevel(xlator_t *this, int log_level) active = ctx->active; top = active->first; - if (strcmp(top->type, "protocol/server") || (log_level == -1)) + if (log_level == -1) return; - /* Set log-level for server xlator */ - top->loglevel = log_level; + if (ctx->cmd_args.brick_mux) { + /* Set log-level for all brick xlators */ + top->loglevel = log_level; - /* Set log-level for parent xlator */ - if (this->parents) - this->parents->xlator->loglevel = log_level; + /* Set log-level for parent xlator */ + if (this->parents) + this->parents->xlator->loglevel = log_level; - while (trav) { - trav->loglevel = log_level; - trav = trav->next; + while (trav) { + trav->loglevel = log_level; + trav = trav->next; + } + } else { + gf_log_set_loglevel(this->ctx, log_level); } } diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 8bbd795c6ab..e6f2d3d3f3c 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -2229,6 +2229,9 @@ retry: if (volinfo->memory_accounting) runner_add_arg(&runner, "--mem-accounting"); + if (is_brick_mx_enabled()) + runner_add_arg(&runner, "--brick-mux"); + runner_log(&runner, "", 0, "Starting GlusterFS"); brickinfo->port = port; @@ -2367,6 +2370,10 @@ unsafe_option(dict_t *this, char *key, data_t *value, void *arg) return _gf_false; } + if (fnmatch("*diagnostics.client-log*", key, 0) == 0) { + return _gf_false; + } + return _gf_true; } -- cgit