From 673f2f90f35e5ea381bd22a43403278f89c957a5 Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Mon, 16 Oct 2017 14:24:29 +0530 Subject: Revert "glusterd: (storhaug) remove ganesha (843e1b0)" please note as an additional change, macro GLUSTERD_GET_SNAP_DIR moved from glusterd-store.c to glusterd-snapshot-utils.h Change-Id: I811efefc148453fe32e4f0d322e80455447cec71 updates: #663 Signed-off-by: Jiffin Tony Thottan --- cli/src/cli-cmd-global.c | 57 ++ cli/src/cli-cmd-parser.c | 118 +++ cli/src/cli-cmd.c | 3 +- cli/src/cli-rpc-ops.c | 82 ++ cli/src/cli.h | 4 + extras/ganesha/scripts/dbus-send.sh | 14 +- extras/ganesha/scripts/ganesha-ha.sh | 27 +- xlators/mgmt/glusterd/src/Makefile.am | 4 +- xlators/mgmt/glusterd/src/glusterd-errno.h | 2 +- xlators/mgmt/glusterd/src/glusterd-ganesha.c | 907 +++++++++++++++++++++ xlators/mgmt/glusterd/src/glusterd-handler.c | 79 ++ xlators/mgmt/glusterd/src/glusterd-messages.h | 3 +- xlators/mgmt/glusterd/src/glusterd-op-sm.c | 46 +- .../mgmt/glusterd/src/glusterd-snapshot-utils.c | 196 +++++ .../mgmt/glusterd/src/glusterd-snapshot-utils.h | 10 + xlators/mgmt/glusterd/src/glusterd-store.c | 10 - xlators/mgmt/glusterd/src/glusterd-store.h | 2 + xlators/mgmt/glusterd/src/glusterd-volume-ops.c | 46 ++ xlators/mgmt/glusterd/src/glusterd-volume-set.c | 7 + xlators/mgmt/glusterd/src/glusterd.h | 23 + 20 files changed, 1608 insertions(+), 32 deletions(-) create mode 100644 xlators/mgmt/glusterd/src/glusterd-ganesha.c diff --git a/cli/src/cli-cmd-global.c b/cli/src/cli-cmd-global.c index d7c4d97ad72..ba3c1ea6446 100644 --- a/cli/src/cli-cmd-global.c +++ b/cli/src/cli-cmd-global.c @@ -36,6 +36,10 @@ int cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word, const char **words, int wordcount); +int +cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word, + const char **words, int wordcount); + struct cli_cmd global_cmds[] = { { "global help", @@ -48,6 +52,11 @@ struct cli_cmd global_cmds[] = { cli_cmd_get_state_cbk, "Get local state representation of mentioned daemon", }, + { + "nfs-ganesha {enable| disable} ", + cli_cmd_ganesha_cbk, + "Enable/disable NFS-Ganesha support", + }, {NULL, NULL, NULL}}; int @@ -88,6 +97,54 @@ out: return ret; } +int +cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word, + const char **words, int wordcount) + +{ + int sent = 0; + int parse_error = 0; + int ret = -1; + rpc_clnt_procedure_t *proc = NULL; + call_frame_t *frame = NULL; + dict_t *options = NULL; + cli_local_t *local = NULL; + char *op_errstr = NULL; + + proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GANESHA]; + + frame = create_frame(THIS, THIS->ctx->pool); + if (!frame) + goto out; + + ret = cli_cmd_ganesha_parse(state, words, wordcount, &options, &op_errstr); + if (ret) { + if (op_errstr) { + cli_err("%s", op_errstr); + GF_FREE(op_errstr); + } else + cli_usage_out(word->pattern); + parse_error = 1; + goto out; + } + + CLI_LOCAL_INIT(local, words, frame, options); + + if (proc->fn) { + ret = proc->fn(frame, THIS, options); + } + +out: + if (ret) { + cli_cmd_sent_status_get(&sent); + if ((sent == 0) && (parse_error == 0)) + cli_out("Setting global option failed"); + } + + CLI_STACK_DESTROY(frame); + return ret; +} + int cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word, const char **words, int wordcount) diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c index decdd10cb50..18aef217a9a 100644 --- a/cli/src/cli-cmd-parser.c +++ b/cli/src/cli-cmd-parser.c @@ -5772,3 +5772,121 @@ out: return ret; } + +/* Parsing global option for NFS-Ganesha config + * gluster nfs-ganesha enable/disable */ + +int32_t +cli_cmd_ganesha_parse(struct cli_state *state, const char **words, + int wordcount, dict_t **options, char **op_errstr) +{ + dict_t *dict = NULL; + int ret = -1; + char *key = NULL; + char *value = NULL; + char *w = NULL; + char *opwords[] = {"enable", "disable", NULL}; + const char *question = NULL; + gf_answer_t answer = GF_ANSWER_NO; + + GF_ASSERT(words); + GF_ASSERT(options); + + dict = dict_new(); + + if (!dict) + goto out; + + if (wordcount != 2) + goto out; + + key = (char *)words[0]; + value = (char *)words[1]; + + if (!key || !value) { + cli_out("Usage : nfs-ganesha "); + ret = -1; + goto out; + } + + ret = gf_strip_whitespace(value, strlen(value)); + if (ret == -1) + goto out; + + if (strcmp(key, "nfs-ganesha")) { + gf_asprintf(op_errstr, + "Global option: error: ' %s '" + "is not a valid global option.", + key); + ret = -1; + goto out; + } + + w = str_getunamb(value, opwords); + if (!w) { + cli_out( + "Invalid global option \n" + "Usage : nfs-ganesha "); + ret = -1; + goto out; + } + + if (strcmp(value, "enable") == 0) { + question = + "Enabling NFS-Ganesha requires Gluster-NFS to be " + "disabled across the trusted pool. Do you " + "still want to continue?\n"; + } else if (strcmp(value, "disable") == 0) { + question = + "Disabling NFS-Ganesha will tear down the entire " + "ganesha cluster across the trusted pool. Do you " + "still want to continue?\n"; + } else { + ret = -1; + goto out; + } + answer = cli_cmd_get_confirmation(state, question); + if (GF_ANSWER_NO == answer) { + gf_log("cli", GF_LOG_ERROR, + "Global operation " + "cancelled, exiting"); + ret = -1; + goto out; + } + cli_out("This will take a few minutes to complete. Please wait .."); + + ret = dict_set_str(dict, "key", key); + if (ret) { + gf_log(THIS->name, GF_LOG_ERROR, "dict set on key failed"); + goto out; + } + + ret = dict_set_str(dict, "value", value); + if (ret) { + gf_log(THIS->name, GF_LOG_ERROR, "dict set on value failed"); + goto out; + } + + ret = dict_set_str(dict, "globalname", "All"); + if (ret) { + gf_log(THIS->name, GF_LOG_ERROR, + "dict set on global" + " key failed."); + goto out; + } + + ret = dict_set_int32(dict, "hold_global_locks", _gf_true); + if (ret) { + gf_log(THIS->name, GF_LOG_ERROR, + "dict set on global key " + "failed."); + goto out; + } + + *options = dict; +out: + if (ret) + dict_unref(dict); + + return ret; +} diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c index 2ee8b1b4968..8c069055b64 100644 --- a/cli/src/cli-cmd.c +++ b/cli/src/cli-cmd.c @@ -366,7 +366,8 @@ cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame, unsigned timeout = 0; if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) || - (GLUSTER_CLI_HEAL_VOLUME == procnum)) + (GLUSTER_CLI_HEAL_VOLUME == procnum) || + (GLUSTER_CLI_GANESHA == procnum)) timeout = cli_ten_minutes_timeout; else timeout = cli_default_conn_timeout; diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c index dcf714bd862..6cd31d200b1 100644 --- a/cli/src/cli-rpc-ops.c +++ b/cli/src/cli-rpc-ops.c @@ -1927,6 +1927,62 @@ out: return ret; } +int +gf_cli_ganesha_cbk(struct rpc_req *req, struct iovec *iov, int count, + void *myframe) +{ + gf_cli_rsp rsp = { + 0, + }; + int ret = -1; + dict_t *dict = NULL; + + GF_ASSERT(myframe); + + if (-1 == req->rpc_status) { + goto out; + } + + ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp); + if (ret < 0) { + gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR, + "Failed to decode xdr response"); + goto out; + } + + gf_log("cli", GF_LOG_DEBUG, "Received resp to ganesha"); + + dict = dict_new(); + + if (!dict) { + ret = -1; + goto out; + } + + ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict); + if (ret) + goto out; + + if (rsp.op_ret) { + if (strcmp(rsp.op_errstr, "")) + cli_err("nfs-ganesha: failed: %s", rsp.op_errstr); + else + cli_err("nfs-ganesha: failed"); + } + + else { + cli_out("nfs-ganesha : success "); + } + + ret = rsp.op_ret; + +out: + if (dict) + dict_unref(dict); + cli_cmd_broadcast_response(ret); + return ret; +} + char * is_server_debug_xlator(void *myframe) { @@ -4199,6 +4255,31 @@ out: return ret; } +int32_t +gf_cli_ganesha(call_frame_t *frame, xlator_t *this, void *data) +{ + gf_cli_req req = {{ + 0, + }}; + int ret = 0; + dict_t *dict = NULL; + + if (!frame || !this || !data) { + ret = -1; + goto out; + } + + dict = data; + + ret = cli_to_glusterd(&req, frame, gf_cli_ganesha_cbk, + (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_GANESHA, + this, cli_rpc_prog, NULL); +out: + gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret); + + return ret; +} + int32_t gf_cli_set_volume(call_frame_t *frame, xlator_t *this, void *data) { @@ -11345,6 +11426,7 @@ struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = { [GLUSTER_CLI_BITROT] = {"BITROT", gf_cli_bitrot}, [GLUSTER_CLI_GET_STATE] = {"GET_STATE", gf_cli_get_state}, [GLUSTER_CLI_RESET_BRICK] = {"RESET_BRICK", gf_cli_reset_brick}, + [GLUSTER_CLI_GANESHA] = {"GANESHA", gf_cli_ganesha}, }; struct rpc_clnt_program cli_prog = { diff --git a/cli/src/cli.h b/cli/src/cli.h index 4e008ffe878..d1fbc143ae9 100644 --- a/cli/src/cli.h +++ b/cli/src/cli.h @@ -283,6 +283,10 @@ int32_t cli_cmd_volume_set_parse(struct cli_state *state, const char **words, int wordcount, dict_t **options, char **op_errstr); +int32_t +cli_cmd_ganesha_parse(struct cli_state *state, const char **words, + int wordcount, dict_t **options, char **op_errstr); + int32_t cli_cmd_get_state_parse(struct cli_state *state, const char **words, int wordcount, dict_t **options, char **op_errstr); diff --git a/extras/ganesha/scripts/dbus-send.sh b/extras/ganesha/scripts/dbus-send.sh index ec8d9485ad7..9d613a0e7ad 100755 --- a/extras/ganesha/scripts/dbus-send.sh +++ b/extras/ganesha/scripts/dbus-send.sh @@ -41,8 +41,18 @@ string:"EXPORT(Path=/$VOL)" #This function removes an export dynamically(uses the export_id of the export) function dynamic_export_remove() { - removed_id=`cat $GANESHA_DIR/exports/export.$VOL.conf |\ -grep Export_Id | awk -F"[=,;]" '{print$2}'| tr -d '[[:space:]]'` + # Below bash fetch all the export from ShowExport command and search + # export entry based on path and then get its export entry. + # There are two possiblities for path, either entire volume will be + # exported or subdir. It handles both cases. But it remove only first + # entry from the list based on assumption that entry exported via cli + # has lowest export id value + removed_id=$(dbus-send --type=method_call --print-reply --system \ + --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ + org.ganesha.nfsd.exportmgr.ShowExports | grep -B 1 -we \ + "/"$VOL -e "/"$VOL"/" | grep uint16 | awk '{print $2}' \ + | head -1) + dbus-send --print-reply --system \ --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ org.ganesha.nfsd.exportmgr.RemoveExport uint16:$removed_id diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh index 2df98ac5fbf..32af1cab10e 100644 --- a/extras/ganesha/scripts/ganesha-ha.sh +++ b/extras/ganesha/scripts/ganesha-ha.sh @@ -20,6 +20,7 @@ # ensure that the NFS GRACE DBUS signal is sent after the VIP moves to # the new host. +GANESHA_HA_SH=$(realpath $0) HA_NUM_SERVERS=0 HA_SERVERS="" HA_VOL_NAME="gluster_shared_storage" @@ -68,9 +69,9 @@ function find_rhel7_conf done } -if [ -z $CONFFILE ] +if [ -z ${CONFFILE} ] then - find_rhel7_conf $OPTIONS + find_rhel7_conf ${OPTIONS} fi @@ -90,9 +91,9 @@ usage() { determine_service_manager () { - if [ -e "/usr/bin/systemctl" ]; + if [ -e "/bin/systemctl" ]; then - SERVICE_MAN="/usr/bin/systemctl" + SERVICE_MAN="/bin/systemctl" elif [ -e "/sbin/invoke-rc.d" ]; then SERVICE_MAN="/sbin/invoke-rc.d" @@ -100,7 +101,7 @@ determine_service_manager () { then SERVICE_MAN="/sbin/service" fi - if [ "$SERVICE_MAN" == "DISTRO_NOT_FOUND" ] + if [ "${SERVICE_MAN}" == "DISTRO_NOT_FOUND" ] then echo "Service manager not recognized, exiting" exit 1 @@ -113,21 +114,21 @@ manage_service () local new_node=${2} local option= - if [ "$action" == "start" ]; then + if [ "${action}" == "start" ]; then option="yes" else option="no" fi ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ -${SECRET_PEM} root@${new_node} "/usr/libexec/ganesha/ganesha-ha.sh --setup-ganesha-conf-files $HA_CONFDIR $option" +${SECRET_PEM} root@${new_node} "${GANESHA_HA_SH} --setup-ganesha-conf-files $HA_CONFDIR $option" - if [ "$SERVICE_MAN" == "/usr/bin/systemctl" ] + if [ "${SERVICE_MAN}" == "/bin/systemctl" ] then ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ -${SECRET_PEM} root@${new_node} "$SERVICE_MAN ${action} nfs-ganesha" +${SECRET_PEM} root@${new_node} "${SERVICE_MAN} ${action} nfs-ganesha" else ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ -${SECRET_PEM} root@${new_node} "$SERVICE_MAN nfs-ganesha ${action}" +${SECRET_PEM} root@${new_node} "${SERVICE_MAN} nfs-ganesha ${action}" fi } @@ -274,8 +275,7 @@ string:\"EXPORT(Export_Id=$export_id)\" 2>&1") ret=$? logger <<< "${output}" if [ ${ret} -ne 0 ]; then - echo "Error: refresh-config failed on ${current_host}." - exit 1 + echo "Refresh-config failed on ${current_host}. Please check logs on ${current_host}" else echo "Refresh-config completed on ${current_host}." fi @@ -296,8 +296,7 @@ string:"EXPORT(Export_Id=$export_id)" 2>&1) ret=$? logger <<< "${output}" if [ ${ret} -ne 0 ] ; then - echo "Error: refresh-config failed on localhost." - exit 1 + echo "Refresh-config failed on localhost." else echo "Success: refresh-config completed." fi diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am index 3b051a0fec9..fe90665ab8f 100644 --- a/xlators/mgmt/glusterd/src/Makefile.am +++ b/xlators/mgmt/glusterd/src/Makefile.am @@ -25,7 +25,7 @@ glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \ glusterd-conn-helper.c glusterd-snapd-svc.c glusterd-snapd-svc-helper.c \ glusterd-bitd-svc.c glusterd-scrub-svc.c glusterd-server-quorum.c \ glusterd-reset-brick.c glusterd-shd-svc.c glusterd-shd-svc-helper.c \ - glusterd-gfproxyd-svc.c glusterd-gfproxyd-svc-helper.c + glusterd-gfproxyd-svc.c glusterd-gfproxyd-svc-helper.c glusterd-ganesha.c glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ @@ -55,6 +55,8 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ -I$(CONTRIBDIR)/mount -I$(CONTRIBDIR)/userspace-rcu \ -DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \ -DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\" \ + -DCONFDIR=\"$(localstatedir)/run/gluster/shared_storage/nfs-ganesha\" \ + -DGANESHA_PREFIX=\"$(libexecdir)/ganesha\" \ -DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) \ -I$(top_srcdir)/libglusterd/src/ diff --git a/xlators/mgmt/glusterd/src/glusterd-errno.h b/xlators/mgmt/glusterd/src/glusterd-errno.h index 7e1575b57af..c74070e0e8d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-errno.h +++ b/xlators/mgmt/glusterd/src/glusterd-errno.h @@ -27,7 +27,7 @@ enum glusterd_op_errno { EG_ISSNAP = 30813, /* Volume is a snap volume */ EG_GEOREPRUN = 30814, /* Geo-Replication is running */ EG_NOTTHINP = 30815, /* Bricks are not thinly provisioned */ - EG_NOGANESHA = 30816, /* obsolete ganesha is not enabled */ + EG_NOGANESHA = 30816, /* Global ganesha is not enabled */ }; #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c new file mode 100644 index 00000000000..0a16925c19b --- /dev/null +++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c @@ -0,0 +1,907 @@ +/* + Copyright (c) 2015 Red Hat, Inc. + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include +#include "glusterd.h" +#include "glusterd-op-sm.h" +#include "glusterd-store.h" +#include "glusterd-utils.h" +#include "glusterd-nfs-svc.h" +#include "glusterd-volgen.h" +#include "glusterd-messages.h" +#include + +#include + +int +start_ganesha(char **op_errstr); + +typedef struct service_command { + char *binary; + char *service; + int (*action)(struct service_command *, char *); +} service_command; + +/* parsing_ganesha_ha_conf will allocate the returned string + * to be freed (GF_FREE) by the caller + * return NULL if error or not found */ +static char * +parsing_ganesha_ha_conf(const char *key) +{ +#define MAX_LINE 1024 + char scratch[MAX_LINE * 2] = { + 0, + }; + char *value = NULL, *pointer = NULL, *end_pointer = NULL; + FILE *fp; + + fp = fopen(GANESHA_HA_CONF, "r"); + if (fp == NULL) { + gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, + "couldn't open the file %s", GANESHA_HA_CONF); + goto end_ret; + } + while ((pointer = fgets(scratch, MAX_LINE, fp)) != NULL) { + /* Read config file until we get matching "^[[:space:]]*key" */ + if (*pointer == '#') { + continue; + } + while (isblank(*pointer)) { + pointer++; + } + if (strncmp(pointer, key, strlen(key))) { + continue; + } + pointer += strlen(key); + /* key found : if we fail to parse, we'll return an error + * rather than trying next one + * - supposition : conf file is bash compatible : no space + * around the '=' */ + if (*pointer != '=') { + gf_msg(THIS->name, GF_LOG_ERROR, errno, + GD_MSG_GET_CONFIG_INFO_FAILED, "Parsing %s failed at key %s", + GANESHA_HA_CONF, key); + goto end_close; + } + pointer++; /* jump the '=' */ + + if (*pointer == '"' || *pointer == '\'') { + /* dont get the quote */ + pointer++; + } + end_pointer = pointer; + /* stop at the next closing quote or blank/newline */ + do { + end_pointer++; + } while (!(*end_pointer == '\'' || *end_pointer == '"' || + isspace(*end_pointer) || *end_pointer == '\0')); + *end_pointer = '\0'; + + /* got it. copy it and return */ + value = gf_strdup(pointer); + break; + } + +end_close: + fclose(fp); +end_ret: + return value; +} + +static int +sc_systemctl_action(struct service_command *sc, char *command) +{ + runner_t runner = { + 0, + }; + + runinit(&runner); + runner_add_args(&runner, sc->binary, command, sc->service, NULL); + return runner_run(&runner); +} + +static int +sc_service_action(struct service_command *sc, char *command) +{ + runner_t runner = { + 0, + }; + + runinit(&runner); + runner_add_args(&runner, sc->binary, sc->service, command, NULL); + return runner_run(&runner); +} + +static int +manage_service(char *action) +{ + int i = 0; + int ret = 0; + struct service_command sc_list[] = {{.binary = "/bin/systemctl", + .service = "nfs-ganesha", + .action = sc_systemctl_action}, + {.binary = "/sbin/invoke-rc.d", + .service = "nfs-ganesha", + .action = sc_service_action}, + {.binary = "/sbin/service", + .service = "nfs-ganesha", + .action = sc_service_action}, + {.binary = NULL}}; + + while (sc_list[i].binary != NULL) { + ret = sys_access(sc_list[i].binary, X_OK); + if (ret == 0) { + gf_msg_debug(THIS->name, 0, "%s found.", sc_list[i].binary); + return sc_list[i].action(&sc_list[i], action); + } + i++; + } + gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_UNRECOGNIZED_SVC_MNGR, + "Could not %s NFS-Ganesha.Service manager for distro" + " not recognized.", + action); + return ret; +} + +/* + * Check if the cluster is a ganesha cluster or not * + */ +gf_boolean_t +glusterd_is_ganesha_cluster() +{ + int ret = -1; + glusterd_conf_t *priv = NULL; + xlator_t *this = NULL; + gf_boolean_t ret_bool = _gf_false; + + this = THIS; + GF_VALIDATE_OR_GOTO("ganesha", this, out); + priv = this->private; + GF_VALIDATE_OR_GOTO(this->name, priv, out); + + ret = dict_get_str_boolean(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, + _gf_false); + if (ret == _gf_true) { + ret_bool = _gf_true; + gf_msg_debug(this->name, 0, "nfs-ganesha is enabled for the cluster"); + } else + gf_msg_debug(this->name, 0, "nfs-ganesha is disabled for the cluster"); + +out: + return ret_bool; +} + +/* Check if ganesha.enable is set to 'on', that checks if + * a particular volume is exported via NFS-Ganesha */ +gf_boolean_t +glusterd_check_ganesha_export(glusterd_volinfo_t *volinfo) +{ + char *value = NULL; + gf_boolean_t is_exported = _gf_false; + int ret = 0; + + ret = glusterd_volinfo_get(volinfo, "ganesha.enable", &value); + if ((ret == 0) && value) { + if (strcmp(value, "on") == 0) { + gf_msg_debug(THIS->name, 0, + "ganesha.enable set" + " to %s", + value); + is_exported = _gf_true; + } + } + return is_exported; +} + +/* * + * The below function is called as part of commit phase for volume set option + * "ganesha.enable". If the value is "on", it creates export configuration file + * and then export the volume via dbus command. Incase of "off", the volume + * will be already unexported during stage phase, so it will remove the conf + * file from shared storage + */ +int +glusterd_check_ganesha_cmd(char *key, char *value, char **errstr, dict_t *dict) +{ + int ret = 0; + char *volname = NULL; + + GF_ASSERT(key); + GF_ASSERT(value); + GF_ASSERT(dict); + + if ((strcmp(key, "ganesha.enable") == 0)) { + if ((strcmp(value, "on")) && (strcmp(value, "off"))) { + gf_asprintf(errstr, + "Invalid value" + " for volume set command. Use on/off only."); + ret = -1; + goto out; + } + if (strcmp(value, "on") == 0) { + ret = glusterd_handle_ganesha_op(dict, errstr, key, value); + + } else if (is_origin_glusterd(dict)) { + ret = dict_get_str(dict, "volname", &volname); + if (ret) { + gf_msg("glusterd-ganesha", GF_LOG_ERROR, errno, + GD_MSG_DICT_GET_FAILED, "Unable to get volume name"); + goto out; + } + ret = manage_export_config(volname, "off", errstr); + } + } +out: + if (ret) { + gf_msg("glusterd-ganesha", GF_LOG_ERROR, 0, + GD_MSG_NFS_GNS_OP_HANDLE_FAIL, + "Handling NFS-Ganesha" + " op failed."); + } + return ret; +} + +int +glusterd_op_stage_set_ganesha(dict_t *dict, char **op_errstr) +{ + int ret = -1; + char *value = NULL; + char *str = NULL; + glusterd_conf_t *priv = NULL; + xlator_t *this = NULL; + + GF_ASSERT(dict); + this = THIS; + GF_ASSERT(this); + priv = this->private; + GF_ASSERT(priv); + + ret = dict_get_str(dict, "value", &value); + if (value == NULL) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "value not present."); + goto out; + } + /* This dict_get will fail if the user had never set the key before */ + /*Ignoring the ret value and proceeding */ + ret = dict_get_str(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, &str); + if (str ? strcmp(value, str) == 0 : strcmp(value, "disable") == 0) { + gf_asprintf(op_errstr, "nfs-ganesha is already %sd.", value); + ret = -1; + goto out; + } + + if (strcmp(value, "enable") == 0) { + ret = start_ganesha(op_errstr); + if (ret) { + gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_NFS_GNS_START_FAIL, + "Could not start NFS-Ganesha"); + } + } else { + ret = stop_ganesha(op_errstr); + if (ret) + gf_msg_debug(THIS->name, 0, + "Could not stop " + "NFS-Ganesha."); + } + +out: + + if (ret) { + if (!(*op_errstr)) { + *op_errstr = gf_strdup("Error, Validation Failed"); + gf_msg_debug(this->name, 0, "Error, Cannot Validate option :%s", + GLUSTERD_STORE_KEY_GANESHA_GLOBAL); + } else { + gf_msg_debug(this->name, 0, "Error, Cannot Validate option"); + } + } + return ret; +} + +int +glusterd_op_set_ganesha(dict_t *dict, char **errstr) +{ + int ret = 0; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + char *key = NULL; + char *value = NULL; + char *next_version = NULL; + + this = THIS; + GF_ASSERT(this); + GF_ASSERT(dict); + + priv = this->private; + GF_ASSERT(priv); + + ret = dict_get_str(dict, "key", &key); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Couldn't get key in global option set"); + goto out; + } + + ret = dict_get_str(dict, "value", &value); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Couldn't get value in global option set"); + goto out; + } + + ret = glusterd_handle_ganesha_op(dict, errstr, key, value); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NFS_GNS_SETUP_FAIL, + "Initial NFS-Ganesha set up failed"); + ret = -1; + goto out; + } + ret = dict_set_dynstr_with_alloc(priv->opts, + GLUSTERD_STORE_KEY_GANESHA_GLOBAL, value); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED, + "Failed to set" + " nfs-ganesha in dict."); + goto out; + } + ret = glusterd_get_next_global_opt_version_str(priv->opts, &next_version); + if (ret) { + gf_msg_debug(THIS->name, 0, + "Could not fetch " + " global op version"); + goto out; + } + ret = dict_set_str(priv->opts, GLUSTERD_GLOBAL_OPT_VERSION, next_version); + if (ret) + goto out; + + ret = glusterd_store_options(this, priv->opts); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_FAIL, + "Failed to store options"); + goto out; + } + +out: + gf_msg_debug(this->name, 0, "returning %d", ret); + return ret; +} + +/* Following function parse GANESHA_HA_CONF + * The sample file looks like below, + * HA_NAME="ganesha-ha-360" + * HA_VOL_NAME="ha-state" + * HA_CLUSTER_NODES="server1,server2" + * VIP_rhs_1="10.x.x.x" + * VIP_rhs_2="10.x.x.x." */ + +/* Check if the localhost is listed as one of nfs-ganesha nodes */ +gf_boolean_t +check_host_list(void) +{ + glusterd_conf_t *priv = NULL; + char *hostname, *hostlist; + gf_boolean_t ret = _gf_false; + xlator_t *this = NULL; + + this = THIS; + priv = THIS->private; + GF_ASSERT(priv); + + hostlist = parsing_ganesha_ha_conf("HA_CLUSTER_NODES"); + if (hostlist == NULL) { + gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_GET_CONFIG_INFO_FAILED, + "couldn't get HA_CLUSTER_NODES from file %s", GANESHA_HA_CONF); + return _gf_false; + } + + /* Hostlist is a comma separated list now */ + hostname = strtok(hostlist, ","); + while (hostname != NULL) { + ret = gf_is_local_addr(hostname); + if (ret) { + gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NFS_GNS_HOST_FOUND, + "ganesha host found " + "Hostname is %s", + hostname); + break; + } + hostname = strtok(NULL, ","); + } + + GF_FREE(hostlist); + return ret; +} + +int +manage_export_config(char *volname, char *value, char **op_errstr) +{ + runner_t runner = { + 0, + }; + int ret = -1; + + GF_ASSERT(volname); + runinit(&runner); + runner_add_args(&runner, GANESHA_PREFIX "/create-export-ganesha.sh", + CONFDIR, value, volname, NULL); + ret = runner_run(&runner); + + if (ret && op_errstr) + gf_asprintf(op_errstr, + "Failed to create" + " NFS-Ganesha export config file."); + + return ret; +} + +/* Exports and unexports a particular volume via NFS-Ganesha */ +int +ganesha_manage_export(dict_t *dict, char *value, + gf_boolean_t update_cache_invalidation, char **op_errstr) +{ + runner_t runner = { + 0, + }; + int ret = -1; + glusterd_volinfo_t *volinfo = NULL; + dict_t *vol_opts = NULL; + char *volname = NULL; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + gf_boolean_t option = _gf_false; + + runinit(&runner); + this = THIS; + GF_ASSERT(this); + priv = this->private; + + GF_ASSERT(value); + GF_ASSERT(dict); + GF_ASSERT(priv); + + ret = dict_get_str(dict, "volname", &volname); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Unable to get volume name"); + goto out; + } + ret = gf_string2boolean(value, &option); + if (ret == -1) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, + "invalid value."); + goto out; + } + + ret = glusterd_volinfo_find(volname, &volinfo); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, + FMTSTR_CHECK_VOL_EXISTS, volname); + goto out; + } + + ret = glusterd_check_ganesha_export(volinfo); + if (ret && option) { + gf_asprintf(op_errstr, + "ganesha.enable " + "is already 'on'."); + ret = -1; + goto out; + + } else if (!option && !ret) { + gf_asprintf(op_errstr, + "ganesha.enable " + "is already 'off'."); + ret = -1; + goto out; + } + + /* Check if global option is enabled, proceed only then */ + ret = dict_get_str_boolean(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, + _gf_false); + if (ret == -1) { + gf_msg_debug(this->name, 0, + "Failed to get " + "global option dict."); + gf_asprintf(op_errstr, + "The option " + "nfs-ganesha should be " + "enabled before setting ganesha.enable."); + goto out; + } + if (!ret) { + gf_asprintf(op_errstr, + "The option " + "nfs-ganesha should be " + "enabled before setting ganesha.enable."); + ret = -1; + goto out; + } + + /* * + * Create the export file from the node where ganesha.enable "on" + * is executed + * */ + if (option && is_origin_glusterd(dict)) { + ret = manage_export_config(volname, "on", op_errstr); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EXPORT_FILE_CREATE_FAIL, + "Failed to create" + "export file for NFS-Ganesha\n"); + goto out; + } + } + + if (check_host_list()) { + /* Check whether ganesha is running on this node */ + if (manage_service("status")) { + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GANESHA_NOT_RUNNING, + "Export failed, NFS-Ganesha is not running"); + } else { + runner_add_args(&runner, GANESHA_PREFIX "/dbus-send.sh", CONFDIR, + value, volname, NULL); + ret = runner_run(&runner); + if (ret) { + gf_asprintf(op_errstr, + "Dynamic export" + " addition/deletion failed." + " Please see log file for details"); + goto out; + } + } + } + + if (update_cache_invalidation) { + vol_opts = volinfo->dict; + ret = dict_set_dynstr_with_alloc(vol_opts, + "features.cache-invalidation", value); + if (ret) + gf_asprintf(op_errstr, + "Cache-invalidation could not" + " be set to %s.", + value); + ret = glusterd_store_volinfo(volinfo, + GLUSTERD_VOLINFO_VER_AC_INCREMENT); + if (ret) + gf_asprintf(op_errstr, "failed to store volinfo for %s", + volinfo->volname); + } +out: + return ret; +} + +int +tear_down_cluster(gf_boolean_t run_teardown) +{ + int ret = 0; + runner_t runner = { + 0, + }; + struct stat st = { + 0, + }; + DIR *dir = NULL; + struct dirent *entry = NULL; + struct dirent scratch[2] = { + { + 0, + }, + }; + char path[PATH_MAX] = { + 0, + }; + + if (run_teardown) { + runinit(&runner); + runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "teardown", + CONFDIR, NULL); + ret = runner_run(&runner); + /* * + * Remove all the entries in CONFDIR expect ganesha.conf and + * ganesha-ha.conf + */ + dir = sys_opendir(CONFDIR); + if (!dir) { + gf_msg_debug(THIS->name, 0, + "Failed to open directory %s. " + "Reason : %s", + CONFDIR, strerror(errno)); + ret = 0; + goto out; + } + + GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); + while (entry) { + snprintf(path, PATH_MAX, "%s/%s", CONFDIR, entry->d_name); + ret = sys_lstat(path, &st); + if (ret == -1) { + gf_msg_debug(THIS->name, 0, + "Failed to stat entry %s :" + " %s", + path, strerror(errno)); + goto out; + } + + if (strcmp(entry->d_name, "ganesha.conf") == 0 || + strcmp(entry->d_name, "ganesha-ha.conf") == 0) + gf_msg_debug(THIS->name, 0, + " %s is not required" + " to remove", + path); + else if (S_ISDIR(st.st_mode)) + ret = recursive_rmdir(path); + else + ret = sys_unlink(path); + + if (ret) { + gf_msg_debug(THIS->name, 0, + " Failed to remove %s. " + "Reason : %s", + path, strerror(errno)); + } + + gf_msg_debug(THIS->name, 0, "%s %s", + ret ? "Failed to remove" : "Removed", entry->d_name); + GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); + } + + ret = sys_closedir(dir); + if (ret) { + gf_msg_debug(THIS->name, 0, + "Failed to close dir %s. Reason :" + " %s", + CONFDIR, strerror(errno)); + } + } + +out: + return ret; +} + +int +setup_cluster(gf_boolean_t run_setup) +{ + int ret = 0; + runner_t runner = { + 0, + }; + + if (run_setup) { + runinit(&runner); + runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "setup", + CONFDIR, NULL); + ret = runner_run(&runner); + } + return ret; +} + +static int +teardown(gf_boolean_t run_teardown, char **op_errstr) +{ + runner_t runner = { + 0, + }; + int ret = 1; + glusterd_volinfo_t *volinfo = NULL; + glusterd_conf_t *priv = NULL; + dict_t *vol_opts = NULL; + + priv = THIS->private; + + ret = tear_down_cluster(run_teardown); + if (ret == -1) { + gf_asprintf(op_errstr, + "Cleanup of NFS-Ganesha" + " HA config failed."); + goto out; + } + + runinit(&runner); + runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "cleanup", + CONFDIR, NULL); + ret = runner_run(&runner); + if (ret) + gf_msg_debug(THIS->name, 0, + "Could not clean up" + " NFS-Ganesha related config"); + + cds_list_for_each_entry(volinfo, &priv->volumes, vol_list) + { + vol_opts = volinfo->dict; + /* All the volumes exported via NFS-Ganesha will be + unexported, hence setting the appropriate keys */ + ret = dict_set_str(vol_opts, "features.cache-invalidation", "off"); + if (ret) + gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED, + "Could not set features.cache-invalidation " + "to off for %s", + volinfo->volname); + + ret = dict_set_str(vol_opts, "ganesha.enable", "off"); + if (ret) + gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED, + "Could not set ganesha.enable to off for %s", + volinfo->volname); + + ret = glusterd_store_volinfo(volinfo, + GLUSTERD_VOLINFO_VER_AC_INCREMENT); + if (ret) + gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL, + "failed to store volinfo for %s", volinfo->volname); + } +out: + return ret; +} + +int +stop_ganesha(char **op_errstr) +{ + int ret = 0; + runner_t runner = { + 0, + }; + + if (check_host_list()) { + runinit(&runner); + runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", + "--setup-ganesha-conf-files", CONFDIR, "no", NULL); + ret = runner_run(&runner); + if (ret) { + gf_asprintf(op_errstr, + "removal of symlink ganesha.conf " + "in /etc/ganesha failed"); + } + ret = manage_service("stop"); + if (ret) + gf_asprintf(op_errstr, + "NFS-Ganesha service could not" + "be stopped."); + } + return ret; +} + +int +start_ganesha(char **op_errstr) +{ + int ret = -1; + dict_t *vol_opts = NULL; + glusterd_volinfo_t *volinfo = NULL; + glusterd_conf_t *priv = NULL; + runner_t runner = { + 0, + }; + + priv = THIS->private; + GF_ASSERT(priv); + + cds_list_for_each_entry(volinfo, &priv->volumes, vol_list) + { + vol_opts = volinfo->dict; + /* Gluster-nfs has to be disabled across the trusted pool */ + /* before attempting to start nfs-ganesha */ + ret = dict_set_str(vol_opts, NFS_DISABLE_MAP_KEY, "on"); + if (ret) + goto out; + + ret = glusterd_store_volinfo(volinfo, + GLUSTERD_VOLINFO_VER_AC_INCREMENT); + if (ret) { + *op_errstr = gf_strdup( + "Failed to store the " + "Volume information"); + goto out; + } + } + + /* If the nfs svc is not initialized it means that the service is not + * running, hence we can skip the process of stopping gluster-nfs + * service + */ + if (priv->nfs_svc.inited) { + ret = priv->nfs_svc.stop(&(priv->nfs_svc), SIGKILL); + if (ret) { + ret = -1; + gf_asprintf(op_errstr, + "Gluster-NFS service could" + "not be stopped, exiting."); + goto out; + } + } + + if (check_host_list()) { + runinit(&runner); + runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", + "--setup-ganesha-conf-files", CONFDIR, "yes", NULL); + ret = runner_run(&runner); + if (ret) { + gf_asprintf(op_errstr, + "creation of symlink ganesha.conf " + "in /etc/ganesha failed"); + goto out; + } + ret = manage_service("start"); + if (ret) + gf_asprintf(op_errstr, + "NFS-Ganesha failed to start." + "Please see log file for details"); + } + +out: + return ret; +} + +static int +pre_setup(gf_boolean_t run_setup, char **op_errstr) +{ + int ret = 0; + if (run_setup) { + if (!check_host_list()) { + gf_asprintf(op_errstr, + "Running nfs-ganesha setup command " + "from node which is not part of ganesha cluster"); + return -1; + } + } + ret = setup_cluster(run_setup); + if (ret == -1) + gf_asprintf(op_errstr, + "Failed to set up HA " + "config for NFS-Ganesha. " + "Please check the log file for details"); + return ret; +} + +int +glusterd_handle_ganesha_op(dict_t *dict, char **op_errstr, char *key, + char *value) +{ + int32_t ret = -1; + gf_boolean_t option = _gf_false; + + GF_ASSERT(dict); + GF_ASSERT(op_errstr); + GF_ASSERT(key); + GF_ASSERT(value); + + if (strcmp(key, "ganesha.enable") == 0) { + ret = ganesha_manage_export(dict, value, _gf_true, op_errstr); + if (ret < 0) + goto out; + } + + /* It is possible that the key might not be set */ + ret = gf_string2boolean(value, &option); + if (ret == -1) { + gf_asprintf(op_errstr, "Invalid value in key-value pair."); + goto out; + } + + if (strcmp(key, GLUSTERD_STORE_KEY_GANESHA_GLOBAL) == 0) { + /* * + * The set up/teardown of pcs cluster should be performed only + * once. This will done on the node in which the cli command + * 'gluster nfs-ganesha ' got executed. So that + * node should part of ganesha HA cluster + */ + if (option) { + ret = pre_setup(is_origin_glusterd(dict), op_errstr); + if (ret < 0) + goto out; + } else { + ret = teardown(is_origin_glusterd(dict), op_errstr); + if (ret < 0) + goto out; + } + } + +out: + return ret; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index f2f44147ec2..58bb075aa57 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -1755,6 +1755,83 @@ glusterd_op_begin(rpcsvc_request_t *req, glusterd_op_t op, void *ctx, return ret; } +int +__glusterd_handle_ganesha_cmd(rpcsvc_request_t *req) +{ + int32_t ret = -1; + gf_cli_req cli_req = {{ + 0, + }}; + dict_t *dict = NULL; + glusterd_op_t cli_op = GD_OP_GANESHA; + char *op_errstr = NULL; + char err_str[2048] = { + 0, + }; + xlator_t *this = NULL; + + this = THIS; + GF_ASSERT(this); + + GF_ASSERT(req); + + ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); + if (ret < 0) { + snprintf(err_str, sizeof(err_str), + "Failed to decode " + "request received from cli"); + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s", + err_str); + req->rpc_err = GARBAGE_ARGS; + goto out; + } + + if (cli_req.dict.dict_len) { + /* Unserialize the dictionary */ + dict = dict_new(); + if (!dict) { + ret = -1; + goto out; + } + + ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, + &dict); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + "failed to " + "unserialize req-buffer to dictionary"); + snprintf(err_str, sizeof(err_str), + "Unable to decode " + "the command"); + goto out; + } else { + dict->extra_stdfree = cli_req.dict.dict_val; + } + } + + gf_msg_trace(this->name, 0, "Received global option request"); + + ret = glusterd_op_begin_synctask(req, GD_OP_GANESHA, dict); +out: + if (ret) { + if (err_str[0] == '\0') + snprintf(err_str, sizeof(err_str), "Operation failed"); + ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str); + } + if (op_errstr) + GF_FREE(op_errstr); + if (dict) + dict_unref(dict); + + return ret; +} + +int +glusterd_handle_ganesha_cmd(rpcsvc_request_t *req) +{ + return glusterd_big_locked_handler(req, __glusterd_handle_ganesha_cmd); +} + static int __glusterd_handle_reset_volume(rpcsvc_request_t *req) { @@ -6424,6 +6501,8 @@ rpcsvc_actor_t gd_svc_cli_actors[GLUSTER_CLI_MAXVALUE] = { [GLUSTER_CLI_BARRIER_VOLUME] = {"BARRIER_VOLUME", GLUSTER_CLI_BARRIER_VOLUME, glusterd_handle_barrier, NULL, 0, DRC_NA}, + [GLUSTER_CLI_GANESHA] = {"GANESHA", GLUSTER_CLI_GANESHA, + glusterd_handle_ganesha_cmd, NULL, 0, DRC_NA}, [GLUSTER_CLI_GET_VOL_OPT] = {"GET_VOL_OPT", GLUSTER_CLI_GET_VOL_OPT, glusterd_handle_get_vol_opt, NULL, 0, DRC_NA}, [GLUSTER_CLI_BITROT] = {"BITROT", GLUSTER_CLI_BITROT, diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h index 9280c81fee5..f75ba905372 100644 --- a/xlators/mgmt/glusterd/src/glusterd-messages.h +++ b/xlators/mgmt/glusterd/src/glusterd-messages.h @@ -301,6 +301,7 @@ GLFS_MSGID( GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL, GD_MSG_SHD_START_FAIL, GD_MSG_SHD_OBJ_GET_FAIL, GD_MSG_SVC_ATTACH_FAIL, GD_MSG_ATTACH_INFO, GD_MSG_DETACH_INFO, GD_MSG_SVC_DETACH_FAIL, - GD_MSG_RPC_TRANSPORT_GET_PEERNAME_FAIL, GD_MSG_CLUSTER_RC_ENABLE); + GD_MSG_RPC_TRANSPORT_GET_PEERNAME_FAIL, GD_MSG_CLUSTER_RC_ENABLE, + GD_MSG_NFS_GANESHA_DISABLED, GD_MSG_GANESHA_NOT_RUNNING); #endif /* !_GLUSTERD_MESSAGES_H_ */ diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index da0d8c945e2..881ee9cff4f 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -1165,6 +1165,13 @@ glusterd_op_stage_set_volume(dict_t *dict, char **op_errstr) if (ret) goto out; + if ((strcmp(key, "ganesha.enable") == 0) && + (strcmp(value, "off") == 0)) { + ret = ganesha_manage_export(dict, "off", _gf_true, op_errstr); + if (ret) + goto out; + } + ret = glusterd_check_quota_cmd(key, value, errstr, sizeof(errstr)); if (ret) goto out; @@ -1585,6 +1592,20 @@ glusterd_op_stage_reset_volume(dict_t *dict, char **op_errstr) goto out; } + /* * + * If key ganesha.enable is set, then volume should be unexported from + * ganesha server. Also it is a volume-level option, perform only when + * volume name not equal to "all"(in other words if volinfo != NULL) + */ + if (volinfo && (!strcmp(key, "all") || !strcmp(key, "ganesha.enable"))) { + if (glusterd_check_ganesha_export(volinfo)) { + ret = ganesha_manage_export(dict, "off", _gf_true, op_errstr); + if (ret) + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_GNS_RESET_FAIL, + "Could not reset ganesha.enable key"); + } + } + if (strcmp(key, "all")) { exists = glusterd_check_option_exists(key, &key_fixed); if (exists == -1) { @@ -2267,6 +2288,16 @@ glusterd_op_reset_volume(dict_t *dict, char **op_rspstr) } } + if (!strcmp(key, "ganesha.enable") || !strcmp(key, "all")) { + if (glusterd_check_ganesha_export(volinfo) && + is_origin_glusterd(dict)) { + ret = manage_export_config(volname, "off", op_rspstr); + if (ret) + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_GNS_RESET_FAIL, + "Could not reset ganesha.enable key"); + } + } + out: GF_FREE(key_fixed); if (quorum_action) @@ -2837,6 +2868,10 @@ glusterd_op_set_volume(dict_t *dict, char **errstr) } } + ret = glusterd_check_ganesha_cmd(key, value, errstr, dict); + if (ret == -1) + goto out; + if (!is_key_glusterd_hooks_friendly(key)) { ret = glusterd_check_option_exists(key, &key_fixed); GF_ASSERT(ret); @@ -4319,7 +4354,8 @@ glusterd_op_build_payload(dict_t **req, char **op_errstr, dict_t *op_ctx) case GD_OP_SYNC_VOLUME: case GD_OP_COPY_FILE: - case GD_OP_SYS_EXEC: { + case GD_OP_SYS_EXEC: + case GD_OP_GANESHA: { dict_copy(dict, req_dict); } break; @@ -5772,6 +5808,10 @@ glusterd_op_stage_validate(glusterd_op_t op, dict_t *dict, char **op_errstr, ret = glusterd_op_stage_set_volume(dict, op_errstr); break; + case GD_OP_GANESHA: + ret = glusterd_op_stage_set_ganesha(dict, op_errstr); + break; + case GD_OP_RESET_VOLUME: ret = glusterd_op_stage_reset_volume(dict, op_errstr); break; @@ -5902,7 +5942,9 @@ glusterd_op_commit_perform(glusterd_op_t op, dict_t *dict, char **op_errstr, case GD_OP_SET_VOLUME: ret = glusterd_op_set_volume(dict, op_errstr); break; - + case GD_OP_GANESHA: + ret = glusterd_op_set_ganesha(dict, op_errstr); + break; case GD_OP_RESET_VOLUME: ret = glusterd_op_reset_volume(dict, op_errstr); break; diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c index 71f1c78622f..5d8353ca51a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c @@ -3811,6 +3811,148 @@ out: return ret; } +/* * + * Here there are two possibilities, either destination is snaphot or + * clone. In the case of snapshot nfs_ganesha export file will be copied + * to snapdir. If it is clone , then new export file will be created for + * the clone in the GANESHA_EXPORT_DIRECTORY, replacing occurences of + * volname with clonename + */ +int +glusterd_copy_nfs_ganesha_file(glusterd_volinfo_t *src_vol, + glusterd_volinfo_t *dest_vol) +{ + int32_t ret = -1; + char snap_dir[PATH_MAX] = { + 0, + }; + char src_path[PATH_MAX] = { + 0, + }; + char dest_path[PATH_MAX] = { + 0, + }; + char buffer[BUFSIZ] = { + 0, + }; + char *find_ptr = NULL; + char *buff_ptr = NULL; + char *tmp_ptr = NULL; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + struct stat stbuf = { + 0, + }; + FILE *src = NULL; + FILE *dest = NULL; + + this = THIS; + GF_VALIDATE_OR_GOTO("snapshot", this, out); + priv = this->private; + GF_VALIDATE_OR_GOTO(this->name, priv, out); + + GF_VALIDATE_OR_GOTO(this->name, src_vol, out); + GF_VALIDATE_OR_GOTO(this->name, dest_vol, out); + + if (glusterd_check_ganesha_export(src_vol) == _gf_false) { + gf_msg_debug(this->name, 0, + "%s is not exported via " + "NFS-Ganesha. Skipping copy of export conf.", + src_vol->volname); + ret = 0; + goto out; + } + + if (src_vol->is_snap_volume) { + GLUSTERD_GET_SNAP_DIR(snap_dir, src_vol->snapshot, priv); + ret = snprintf(src_path, PATH_MAX, "%s/export.%s.conf", snap_dir, + src_vol->snapshot->snapname); + } else { + ret = snprintf(src_path, PATH_MAX, "%s/export.%s.conf", + GANESHA_EXPORT_DIRECTORY, src_vol->volname); + } + if (ret < 0 || ret >= PATH_MAX) + goto out; + + ret = sys_lstat(src_path, &stbuf); + if (ret) { + /* + * This code path is hit, only when the src_vol is being * + * exported via NFS-Ganesha. So if the conf file is not * + * available, we fail the snapshot operation. * + */ + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, + "Stat on %s failed with %s", src_path, strerror(errno)); + goto out; + } + + if (dest_vol->is_snap_volume) { + memset(snap_dir, 0, PATH_MAX); + GLUSTERD_GET_SNAP_DIR(snap_dir, dest_vol->snapshot, priv); + ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf", + snap_dir, dest_vol->snapshot->snapname); + if (ret < 0) + goto out; + + ret = glusterd_copy_file(src_path, dest_path); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY, + "Failed to copy %s in %s", src_path, dest_path); + goto out; + } + + } else { + ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf", + GANESHA_EXPORT_DIRECTORY, dest_vol->volname); + if (ret < 0) + goto out; + + src = fopen(src_path, "r"); + dest = fopen(dest_path, "w"); + + if (!src || !dest) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, + "Failed to open %s", dest ? src_path : dest_path); + ret = -1; + goto out; + } + + /* * + * if the source volume is snapshot, the export conf file + * consists of orginal volname + */ + if (src_vol->is_snap_volume) + find_ptr = gf_strdup(src_vol->parent_volname); + else + find_ptr = gf_strdup(src_vol->volname); + + if (!find_ptr) + goto out; + + /* Replacing volname with clonename */ + while (fgets(buffer, BUFSIZ, src)) { + buff_ptr = buffer; + while ((tmp_ptr = strstr(buff_ptr, find_ptr))) { + while (buff_ptr < tmp_ptr) + fputc((int)*buff_ptr++, dest); + fputs(dest_vol->volname, dest); + buff_ptr += strlen(find_ptr); + } + fputs(buff_ptr, dest); + memset(buffer, 0, BUFSIZ); + } + } +out: + if (src) + fclose(src); + if (dest) + fclose(dest); + if (find_ptr) + GF_FREE(find_ptr); + + return ret; +} + int32_t glusterd_restore_geo_rep_files(glusterd_volinfo_t *snap_vol) { @@ -3899,6 +4041,60 @@ out: return ret; } +int +glusterd_restore_nfs_ganesha_file(glusterd_volinfo_t *src_vol, + glusterd_snap_t *snap) +{ + int32_t ret = -1; + char snap_dir[PATH_MAX] = ""; + char src_path[PATH_MAX] = ""; + char dest_path[PATH_MAX] = ""; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + struct stat stbuf = { + 0, + }; + + this = THIS; + GF_VALIDATE_OR_GOTO("snapshot", this, out); + priv = this->private; + GF_VALIDATE_OR_GOTO(this->name, priv, out); + + GF_VALIDATE_OR_GOTO(this->name, src_vol, out); + GF_VALIDATE_OR_GOTO(this->name, snap, out); + + GLUSTERD_GET_SNAP_DIR(snap_dir, snap, priv); + + ret = snprintf(src_path, sizeof(src_path), "%s/export.%s.conf", snap_dir, + snap->snapname); + if (ret < 0) + goto out; + + ret = sys_lstat(src_path, &stbuf); + if (ret) { + if (errno == ENOENT) { + ret = 0; + gf_msg_debug(this->name, 0, "%s not found", src_path); + } else + gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED, + "Stat on %s failed with %s", src_path, strerror(errno)); + goto out; + } + + ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf", + GANESHA_EXPORT_DIRECTORY, src_vol->volname); + if (ret < 0) + goto out; + + ret = glusterd_copy_file(src_path, dest_path); + if (ret) + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY, + "Failed to copy %s in %s", src_path, dest_path); + +out: + return ret; +} + /* Snapd functions */ int glusterd_is_snapd_enabled(glusterd_volinfo_t *volinfo) diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h index 19fedecee8d..5762999bba7 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h @@ -10,6 +10,16 @@ #ifndef _GLUSTERD_SNAP_UTILS_H #define _GLUSTERD_SNAP_UTILS_H +#define GLUSTERD_GET_SNAP_DIR(path, snap, priv) \ + do { \ + int32_t _snap_dir_len; \ + _snap_dir_len = snprintf(path, PATH_MAX, "%s/snaps/%s", priv->workdir, \ + snap->snapname); \ + if ((_snap_dir_len < 0) || (_snap_dir_len >= PATH_MAX)) { \ + path[0] = 0; \ + } \ + } while (0) + int32_t glusterd_snap_volinfo_find(char *volname, glusterd_snap_t *snap, glusterd_volinfo_t **volinfo); diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index b7246e2dad5..50773fb1de8 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -48,16 +48,6 @@ #include "mntent_compat.h" #endif -#define GLUSTERD_GET_SNAP_DIR(path, snap, priv) \ - do { \ - int32_t _snap_dir_len; \ - _snap_dir_len = snprintf(path, PATH_MAX, "%s/snaps/%s", priv->workdir, \ - snap->snapname); \ - if ((_snap_dir_len < 0) || (_snap_dir_len >= PATH_MAX)) { \ - path[0] = 0; \ - } \ - } while (0) - #define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv) \ do { \ int32_t _brick_len; \ diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h index 45aba64ff8d..04070549678 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.h +++ b/xlators/mgmt/glusterd/src/glusterd-store.h @@ -110,6 +110,8 @@ typedef enum glusterd_store_ver_ac_ { #define GLUSTERD_STORE_KEY_VOL_MIGRATIONS_SKIPPED "migration-skipped" #define GLUSTERD_STORE_KEY_VOL_MIGRATION_RUN_TIME "migration-run-time" +#define GLUSTERD_STORE_KEY_GANESHA_GLOBAL "nfs-ganesha" + int32_t glusterd_store_volinfo(glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac); diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c index cd92a16c0f7..b21c3966e5c 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c @@ -1628,6 +1628,18 @@ glusterd_op_stage_stop_volume(dict_t *dict, char **op_errstr) goto out; } + ret = glusterd_check_ganesha_export(volinfo); + if (ret) { + ret = ganesha_manage_export(dict, "off", _gf_false, op_errstr); + if (ret) { + gf_msg(THIS->name, GF_LOG_WARNING, 0, + GD_MSG_NFS_GNS_UNEXPRT_VOL_FAIL, + "Could not " + "unexport volume via NFS-Ganesha"); + ret = 0; + } + } + if (glusterd_is_defrag_on(volinfo)) { snprintf(msg, sizeof(msg), "rebalance session is " @@ -2503,6 +2515,8 @@ glusterd_op_start_volume(dict_t *dict, char **op_errstr) xlator_t *this = NULL; glusterd_conf_t *conf = NULL; glusterd_svc_t *svc = NULL; + char *str = NULL; + gf_boolean_t option = _gf_false; this = THIS; GF_ASSERT(this); @@ -2560,6 +2574,29 @@ glusterd_op_start_volume(dict_t *dict, char **op_errstr) } } + ret = dict_get_str(conf->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, &str); + if (ret != 0) { + gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED, + "Global dict not present."); + ret = 0; + + } else { + ret = gf_string2boolean(str, &option); + /* Check if the feature is enabled and set nfs-disable to true */ + if (option) { + gf_msg_debug(this->name, 0, "NFS-Ganesha is enabled"); + /* Gluster-nfs should not start when NFS-Ganesha is enabled*/ + ret = dict_set_str(volinfo->dict, NFS_DISABLE_MAP_KEY, "on"); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Failed to set nfs.disable for" + "volume %s", + volname); + goto out; + } + } + } + ret = glusterd_start_volume(volinfo, flags, _gf_true); if (ret) goto out; @@ -2699,6 +2736,15 @@ glusterd_op_delete_volume(dict_t *dict) goto out; } + if (glusterd_check_ganesha_export(volinfo) && is_origin_glusterd(dict)) { + ret = manage_export_config(volname, "off", NULL); + if (ret) + gf_msg(this->name, GF_LOG_WARNING, 0, 0, + "Could not delete ganesha export conf file " + "for %s", + volname); + } + ret = glusterd_delete_volume(volinfo); out: gf_msg_debug(this->name, 0, "returning %d", ret); diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index 4cadcd5dd19..2ed8d48be59 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -2609,6 +2609,13 @@ struct volopt_map_entry glusterd_volopt_map[] = { .voltype = "features/upcall", .op_version = GD_OP_VERSION_3_7_0, }, + { + .key = "ganesha.enable", + .voltype = "mgmt/ganesha", + .value = "off", + .option = "ganesha.enable", + .op_version = GD_OP_VERSION_7_0, + }, /* Lease translator options */ { .key = "features.leases", diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 899bd0e357b..067e75000b1 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -65,6 +65,9 @@ #define GLUSTERD_LOCALTIME_LOGGING_KEY "cluster.localtime-logging" #define GLUSTERD_DAEMON_LOG_LEVEL_KEY "cluster.daemon-log-level" +#define GANESHA_HA_CONF CONFDIR "/ganesha-ha.conf" +#define GANESHA_EXPORT_DIRECTORY CONFDIR "/exports" + #define GLUSTERD_SNAPS_MAX_HARD_LIMIT 256 #define GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT 90 #define GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT 100 @@ -1177,6 +1180,26 @@ glusterd_op_stop_volume(dict_t *dict); int glusterd_op_delete_volume(dict_t *dict); int +glusterd_handle_ganesha_op(dict_t *dict, char **op_errstr, char *key, + char *value); +int +glusterd_check_ganesha_cmd(char *key, char *value, char **errstr, dict_t *dict); +int +glusterd_op_stage_set_ganesha(dict_t *dict, char **op_errstr); +int +glusterd_op_set_ganesha(dict_t *dict, char **errstr); +int +ganesha_manage_export(dict_t *dict, char *value, + gf_boolean_t update_cache_invalidation, char **op_errstr); +gf_boolean_t +glusterd_is_ganesha_cluster(); +gf_boolean_t +glusterd_check_ganesha_export(glusterd_volinfo_t *volinfo); +int +stop_ganesha(char **op_errstr); +int +tear_down_cluster(gf_boolean_t run_teardown); +int manage_export_config(char *volname, char *value, char **op_errstr); int -- cgit