NFS-Ganesha: Handling CLI commands when NFS-Ganesha keys are set

When ganesha.enable is set to on and features.ganesha is
enabled, there are a few behaviour changes that should
be seen in other volume operations.

1. ganesha.enable can be set to 'on' only
when features.ganesha is set to 'enable'

2.When gluster vol is started, and if ganesha.enable
key was set to 'on', it should automatically export the volume
via NFS-Ganesha.

3.When ganesha.enable is set to 'on', and a volume
is stopped, that volume should be unexported via NFS-Ganesha.

4. gluster vol reset <volname>
If ganesha.enable was set to on, then unexport the
volume via NFS-Ganesha.

5. gluster vol reset all
If features.ganesha is set to enable, as part
of reset all, set it to disable. This translates
to teardown cluster.

All the above problems are fixed by checking the global key
and value, depending on the value, specific functions are called.
And also, functions related to global commands
are moved to cli-cmd-global.c

Commit phase of features.ganesha enable/disable
runs the ganesha-ha.sh setup/teardown respectively.
Before the script begins, it is important that the
NFS-Ganesha service starts on all the HA nodes.
Having the start service commands in the
commit phase could lead to problems.
Moving the pre-requisite service start
commands to the 'stage' phase.

Change-Id: I5a256f94f8e1310ddcd5369f329b7168b2a24c47
BUG: 1200265
Signed-off-by: Meghana Madhusudhan <mmadhusu@redhat.com>
Reviewed-on: http://review.gluster.org/10283
Reviewed-by: jiffin tony Thottan <jthottan@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

6 files changed, 238 insertions, 68 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
index fe67be187f2..b4375aa26c2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c
+++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
@@ -20,8 +20,13 @@
 #include "glusterd-store.h"
 #include "glusterd-utils.h"
 #include "glusterd-nfs-svc.h"
+#include "glusterd-volgen.h"
 #define MAXBUF 1024
 #define DELIM "=\""
+#define SHARED_STORAGE_MNT "/var/run/gluster/shared_storage/nfs-ganesha"
+
+int start_ganesha (char **op_errstr);
+
 
 typedef struct service_command {
         char *binary;
@@ -91,12 +96,31 @@ manage_service (char *action)
                 " not recognized.", action);
         return ret;
 }
+/* Check if ganesha.enable is set to 'on', that checks if
+ * a  particular volume is exported via NFS-Ganesha */
+gf_boolean_t
+glusterd_check_ganesha_export (glusterd_volinfo_t *volinfo) {
+
+        char *value              = NULL;
+        gf_boolean_t is_exported = _gf_false;
+        int ret                 = 0;
+
+        ret = glusterd_volinfo_get (volinfo, "ganesha.enable", &value);
+        if ((ret == 0) && value) {
+                if (strcmp (value, "on") == 0) {
+                        gf_log (THIS->name, GF_LOG_DEBUG, "ganesha.enable set"
+                                " to %s", value);
+                        is_exported = _gf_true;
+                }
+        }
+        return is_exported;
+}
+
 
 int
 glusterd_check_ganesha_cmd (char *key, char *value, char **errstr, dict_t *dict)
 {
         int                ret = 0;
-        gf_boolean_t       b   = _gf_false;
         xlator_t                *this = NULL;
 
         this = THIS;
@@ -104,24 +128,17 @@ glusterd_check_ganesha_cmd (char *key, char *value, char **errstr, dict_t *dict)
         GF_ASSERT (key);
         GF_ASSERT (value);
 
-        if ((strcmp (key, "ganesha.enable") == 0) ||
-           (strcmp (key, "features.ganesha") == 0)) {
-                ret = gf_string2boolean (value, &b);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to parse bool"
-                                "string");
-                        goto out;
-                }
+        if ((strcmp (key, "ganesha.enable") == 0)) {
                 if ((strcmp (value, "on")) && (strcmp (value, "off"))) {
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid value"
-                                "for volume set command. Use on/off only");
+                        gf_asprintf (errstr, "Invalid value"
+                                " for volume set command. Use on/off only.");
                         ret = -1;
                         goto out;
                 }
                 ret = glusterd_handle_ganesha_op (dict, errstr, key, value);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Handling NFS-Ganesha op"
-                                "failed.");
+                        gf_log (this->name, GF_LOG_ERROR, "Handling NFS-Ganesha"
+                                " op failed.");
                 }
         }
 out:
@@ -134,9 +151,9 @@ glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr)
         int                             ret                     = -1;
         char                            *volname                = NULL;
         int                             exists                  = 0;
-        char                            *key                    = NULL;
-        char                            *value                  = NULL;
-        char                            str[100]                = {0, } ;
+        gf_boolean_t                    value                   = _gf_false;
+        gf_boolean_t                    option                  = _gf_false;
+        char                            *str                    = NULL;
         int                             dict_count              = 0;
         int                             flags                   = 0;
         char                            errstr[2048]            = {0, } ;
@@ -150,27 +167,46 @@ glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr)
         priv = this->private;
         GF_ASSERT (priv);
 
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                       "invalid key");
+        value = dict_get_str_boolean (dict, "value", _gf_false);
+        if (value == -1) {
+               gf_log (this->name, GF_LOG_ERROR,
+                        "value not present.");
                 goto out;
         }
-
-        ret = dict_get_str (dict, "value", &value);
-        if (ret) {
-               gf_log (this->name, GF_LOG_ERROR,
-                        "invalid key,value pair in 'global vol set'");
+        /* This dict_get will fail if the user had never set the key before */
+        /*Ignoring the ret value and proceeding */
+        ret = dict_get_str (priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, &str);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "Global dict not present.");
+                ret = 0;
                 goto out;
         }
+        /* Validity of the value is already checked */
+        ret = gf_string2boolean (str, &option);
+        /* Check if the feature is already enabled, fail in that case */
+        if (value == option) {
+                gf_asprintf (op_errstr, "nfs-ganesha is already %sd.", str);
+                ret = -1;
+                goto out;
+        }
+
+        if (value) {
+                ret =  start_ganesha (op_errstr);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Could not start NFS-Ganesha");
+
+                }
+        }
+
 out:
 
         if (ret) {
                 if (!(*op_errstr)) {
                         *op_errstr = gf_strdup ("Error, Validation Failed");
                         gf_log (this->name, GF_LOG_DEBUG,
-                                "Error, Cannot Validate option :%s %s",
-                                key, value);
+                                "Error, Cannot Validate option :%s",
+                                GLUSTERD_STORE_KEY_GANESHA_GLOBAL);
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "Error, Cannot Validate option");
@@ -194,7 +230,6 @@ glusterd_op_set_ganesha (dict_t *dict, char **errstr)
         int32_t                                  dict_count = 0;
         dict_t                                  *vol_opts = NULL;
         int count                                = 0;
-        char *dup                                = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -218,12 +253,6 @@ glusterd_op_set_ganesha (dict_t *dict, char **errstr)
                 goto out;
         }
 
-        dup = gf_strdup (value);
-        if (!dup) {
-                ret = -1;
-                goto out;
-        }
-
         ret = glusterd_handle_ganesha_op (dict, errstr, key, value);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -231,10 +260,12 @@ glusterd_op_set_ganesha (dict_t *dict, char **errstr)
                 ret = -1;
                 goto out;
         }
-        ret = dict_set_str(priv->opts, "features.ganesha", value);
+        ret = dict_set_dynstr_with_alloc (priv->opts,
+                                   GLUSTERD_STORE_KEY_GANESHA_GLOBAL,
+                                   value);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "Failed to set"
-                        " features.ganesha in dict.");
+                        " nfs-ganesha in dict.");
                 goto out;
         }
 
@@ -380,6 +411,7 @@ create_export_config (char *volname, char **op_errstr)
         return ret;
 }
 
+/* Exports and unexports a particular volume via NFS-Ganesha */
 int
 ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
 {
@@ -389,13 +421,18 @@ ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
         glusterd_volinfo_t      *volinfo = NULL;
         char                    *volname = NULL;
         xlator_t                *this    = NULL;
+        glusterd_conf_t         *priv    = NULL;
+        gf_boolean_t             option  = _gf_false;
         int                     i = 1;
 
         runinit (&runner);
         this =  THIS;
+        GF_ASSERT (this);
+        priv = this->private;
 
         GF_ASSERT (value);
         GF_ASSERT (dict);
+        GF_ASSERT (priv);
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
@@ -403,6 +440,11 @@ ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
                         "Unable to get volume name");
                 goto out;
         }
+        ret = gf_string2boolean (value, &option);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "invalid value.");
+                goto out;
+        }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
@@ -410,10 +452,42 @@ ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
                         FMTSTR_CHECK_VOL_EXISTS, volname);
                 goto out;
         }
-        /* Todo : check if global option is enabled, proceed only then */
+
+        ret = glusterd_check_ganesha_export (volinfo);
+        if (ret && option) {
+                gf_asprintf (op_errstr, "ganesha.enable "
+                             "is already 'on'.");
+                ret = -1;
+                goto out;
+
+        }  else if (!option && !ret) {
+                gf_asprintf (op_errstr, "ganesha.enable "
+                                    "is already 'off'.");
+                ret = -1;
+                goto out;
+        }
+
+        /* Check if global option is enabled, proceed only then */
+        ret = dict_get_str_boolean (priv->opts,
+                            GLUSTERD_STORE_KEY_GANESHA_GLOBAL, _gf_false);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to get "
+                        "global option dict.");
+                gf_asprintf (op_errstr, "The option "
+                             "nfs-ganesha should be "
+                             "enabled before setting ganesha.enable.");
+                goto out;
+        }
+        if (!ret) {
+                gf_asprintf (op_errstr, "The option "
+                             "nfs-ganesha should be "
+                             "enabled before setting ganesha.enable.");
+                ret = -1;
+                goto out;
+        }
 
         /* Create the export file only when ganesha.enable "on" is executed */
-         if (strcmp (value, "on") == 0) {
+         if (option) {
                 ret  =  create_export_config (volname, op_errstr);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR, "Failed to create"
@@ -468,11 +542,16 @@ setup_cluster(void)
 }
 
 
-int
-stop_ganesha (char **op_errstr)
+static int
+teardown (char **op_errstr)
 {
         runner_t                runner                     = {0,};
         int                     ret                        = 1;
+        glusterd_volinfo_t      *volinfo                   = NULL;
+        glusterd_conf_t         *priv                      = NULL;
+        dict_t                  *vol_opts                  = NULL;
+
+        priv = THIS->private;
 
         ret = tear_down_cluster();
         if (ret == -1) {
@@ -480,6 +559,37 @@ stop_ganesha (char **op_errstr)
                              " HA config failed.");
                 goto out;
         }
+        ret =  stop_ganesha (op_errstr);
+        if (ret) {
+                gf_asprintf (op_errstr, "Could not stop NFS-Ganesha.");
+                goto out;
+        }
+
+        runinit (&runner);
+        runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
+                                 "cleanup", CONFDIR,  NULL);
+        ret = runner_run (&runner);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_DEBUG, "Could not clean up"
+                        " NFS-Ganesha related config");
+
+        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                vol_opts = volinfo->dict;
+                /* All the volumes exported via NFS-Ganesha will be
+                unexported, hence setting the appropriate key */
+                ret = dict_set_str (vol_opts, "ganesha.enable", "off");
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "Could not set ganesha.enable to off");
+        }
+out:
+        return ret;
+}
+
+int
+stop_ganesha (char **op_errstr) {
+
+        int ret = 0;
 
         if (check_host_list ()) {
                 ret = manage_service ("stop");
@@ -487,8 +597,8 @@ stop_ganesha (char **op_errstr)
                         gf_asprintf (op_errstr, "NFS-Ganesha service could not"
                                      "be stopped.");
         }
-out:
         return ret;
+
 }
 
 int
@@ -525,19 +635,34 @@ start_ganesha (char **op_errstr)
 
         if (check_host_list()) {
                 ret = manage_service ("start");
-                if (ret) {
+                if (ret)
                         gf_asprintf (op_errstr, "NFS-Ganesha failed to start."
                         "Please see log file for details");
-                        goto out;
-                }
+        }
 
+out:
+        return ret;
+}
+
+static int
+pre_setup (char **op_errstr)
+{
+        int    ret = 0;
+
+        ret = mkdir (SHARED_STORAGE_MNT, 0775);
+
+        if ((-1 == ret) && (EEXIST != errno)) {
+                gf_log ("THIS->name", GF_LOG_ERROR, "mkdir() failed on path %s,"
+                        "errno: %s", SHARED_STORAGE_MNT, strerror (errno));
+                goto out;
+        }
+
+        if (check_host_list()) {
                 ret = setup_cluster();
-                if (ret == -1) {
+                if (ret == -1)
                         gf_asprintf (op_errstr, "Failed to set up HA "
                                      "config for NFS-Ganesha. "
                                      "Please check the log file for details");
-                        goto out;
-                }
         }
 
 out:
@@ -552,35 +677,38 @@ glusterd_handle_ganesha_op (dict_t *dict, char **op_errstr,
         int32_t                 ret          = -1;
         char                   *volname      = NULL;
         xlator_t               *this         = NULL;
+        gf_boolean_t           option        = _gf_false;
         static int             export_id     = 1;
         glusterd_volinfo_t     *volinfo      = NULL;
-        char *option                         = NULL;
 
         GF_ASSERT (dict);
         GF_ASSERT (op_errstr);
         GF_ASSERT (key);
         GF_ASSERT (value);
 
-        /* TODO: enable only if global option is set */
-        /* BUG ID : 1200265 */
 
         if (strcmp (key, "ganesha.enable") == 0) {
-                ret =  ganesha_manage_export(dict, value, op_errstr);
+                ret =  ganesha_manage_export (dict, value, op_errstr);
                 if (ret < 0)
                         goto out;
         }
 
-        if (strcmp (key, "features.ganesha") == 0) {
-                if (strcmp (value, "enable") == 0) {
-                        ret = start_ganesha(op_errstr);
+        /* It is possible that the key might not be set */
+        ret =  gf_string2boolean (value, &option);
+        if (ret == -1) {
+                gf_asprintf (op_errstr, "Invalid value in key-value pair.");
+                goto out;
+        }
+
+        if (strcmp (key, GLUSTERD_STORE_KEY_GANESHA_GLOBAL) == 0) {
+                if (option) {
+                        ret = pre_setup (op_errstr);
+                        if (ret < 0)
+                                goto out;
+                } else {
+                        ret = teardown (op_errstr);
                         if (ret < 0)
                                 goto out;
-                }
-
-        else if (strcmp (value, "disable") == 0) {
-                ret = stop_ganesha (op_errstr);
-                if (ret < 0)
-                        goto out;
                 }
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 42477ebf6cd..aac393078b5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -1658,6 +1658,8 @@ glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
         gf_boolean_t    all             = _gf_false;
         char            *next_version   = NULL;
         gf_boolean_t    quorum_action   = _gf_false;
+        gf_boolean_t     option         = _gf_false;
+        char            *op_errstr      = NULL;
 
         conf = this->private;
         ret = dict_get_str (dict, "key", &key);
@@ -1684,6 +1686,18 @@ glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
 
         if (key_fixed)
                 key = key_fixed;
+        option = dict_get_str_boolean (conf->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL,
+                            _gf_false);
+        if (option) {
+                ret = tear_down_cluster();
+                if (ret == -1)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "Could not tear down NFS-Ganesha cluster");
+                ret =  stop_ganesha (&op_errstr);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "Could not stop NFS-Ganesha service");
+        }
 
         ret = -1;
         dup_opt = dict_new ();
@@ -1788,6 +1802,15 @@ glusterd_op_reset_volume (dict_t *dict, char **op_rspstr)
 
         if (glusterd_is_quorum_changed (volinfo->dict, key, NULL))
                 quorum_action = _gf_true;
+        ret = glusterd_check_ganesha_export (volinfo);
+        if (ret) {
+                ret = ganesha_manage_export (dict, "off", op_rspstr);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "Could not reset ganesha.enable key");
+                        ret = 0;
+                }
+        }
 
         ret = glusterd_options_reset (volinfo, key, &is_force);
         if (ret == -1) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index 72fdd851d23..7dbd811803a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -83,6 +83,7 @@ typedef enum glusterd_store_ver_ac_{
 #define GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT  "snap-max-soft-limit"
 #define GLUSTERD_STORE_KEY_SNAPD_PORT           "snapd-port"
 #define GLUSTERD_STORE_KEY_SNAP_ACTIVATE        "snap-activate-on-create"
+#define GLUSTERD_STORE_KEY_GANESHA_GLOBAL       "nfs-ganesha"
 
 #define GLUSTERD_STORE_KEY_BRICK_HOSTNAME       "hostname"
 #define GLUSTERD_STORE_KEY_BRICK_PATH           "path"
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index de3045ffde3..3c3e6e01d8e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1506,6 +1506,15 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
                 ret = -1;
                 goto out;
         }
+        ret = glusterd_check_ganesha_export (volinfo);
+        if (ret) {
+                ret = ganesha_manage_export(dict, "off", op_errstr);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_WARNING, "Could not"
+                                        "unexport volume via NFS-Ganesha");
+                        ret = 0;
+                }
+        }
 
         if (glusterd_is_rb_ongoing (volinfo)) {
                 snprintf (msg, sizeof (msg), "Replace brick is in progress on "
@@ -2331,7 +2340,17 @@ glusterd_op_start_volume (dict_t *dict, char **op_errstr)
                 if (ret)
                         goto out;
         }
-
+        /* Check if the volume is exported via NFS-Ganesha, if yes
+         * export it as part of starting the volume */
+        ret = glusterd_check_ganesha_export (volinfo);
+        if (ret) {
+                ret = ganesha_manage_export (dict, "on", op_errstr);
+                if (ret) {
+                        gf_log ("", GF_LOG_WARNING, "NFS-Ganesha couldn't"
+                                "export the volume. %s", *op_errstr);
+                        ret = 0;
+                }
+        }
         ret = glusterd_svcs_manager (volinfo);
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 2194c429657..17f34e6f86d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1756,15 +1756,10 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .op_version = GD_OP_VERSION_3_7_0,
           .flags      = OPT_FLAG_CLIENT_OPT
         },
-        { .key         = "features.ganesha",
-          .voltype     = "features/ganesha",
-          .option      = "!ganesha",
-          .type         = GLOBAL_NO_DOC,
-          .op_version   = GD_OP_VERSION_3_7_0,
-        },
         { .key         = "ganesha.enable",
           .voltype     = "features/ganesha",
-          .type        = NO_DOC,
+          .value       = "off",
+          .option      = "ganesha.enable",
           .op_version  = GD_OP_VERSION_3_7_0,
         },
         { .key        = "features.shard",
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index e70276d0506..f2a9be15c9f 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -998,6 +998,10 @@ int glusterd_check_ganesha_cmd (char *key, char *value,
                                 char **errstr, dict_t *dict);
 int glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr);
 int glusterd_op_set_ganesha (dict_t *dict, char **errstr);
+int ganesha_manage_export (dict_t *dict, char *value, char **op_errstr);
+gf_boolean_t glusterd_check_ganesha_export (glusterd_volinfo_t *volinfo);
+int stop_ganesha (char **op_errstr);
+int tear_down_cluster (void);
 int glusterd_op_add_brick (dict_t *dict, char **op_errstr);
 int glusterd_op_remove_brick (dict_t *dict, char **op_errstr);
 int glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr,