mgmt/glusterd: gnfs support in gluster command line

Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1319 (gnfs support in gluster command line)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1319

7 files changed, 484 insertions, 4 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 1f8bb55ed..ea3e59b68 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -289,6 +289,30 @@ out:
         return ret;
 }
 
+static int
+glusterd_check_generate_start_nfs (glusterd_volinfo_t *volinfo)
+{
+        int ret = -1;
+
+        if (!volinfo) {
+                gf_log ("", GF_LOG_ERROR, "Invalid Arguments");
+                goto out;
+        }
+
+        ret = volgen_generate_nfs_volfile (volinfo);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_nfs_started ()) {
+                ret = glusterd_nfs_server_stop ();
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_nfs_server_start ();
+out:
+        return ret;
+}
 
 static int
 glusterd_op_stage_create_volume (gd1_mgmt_stage_op_req *req)
@@ -1181,7 +1205,8 @@ glusterd_op_add_brick (gd1_mgmt_stage_op_req *req)
         if (ret)
                 goto out;
 
-
+        if (GLUSTERD_STATUS_STARTED == volinfo->status)
+                ret = glusterd_check_generate_start_nfs (volinfo);
 
 out:
         if (dict)
@@ -2151,7 +2176,8 @@ glusterd_op_remove_brick (gd1_mgmt_stage_op_req *req)
         if (ret)
                 goto out;
 
-
+        if (GLUSTERD_STATUS_STARTED == volinfo->status)
+                ret = glusterd_check_generate_start_nfs (volinfo);
 
 out:
         if (dict)
@@ -2247,6 +2273,10 @@ glusterd_op_start_volume (gd1_mgmt_stage_op_req *req)
                 goto out;
 
         ret = glusterd_volume_compute_cksum (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_check_generate_start_nfs (volinfo);
 
 out:
         return ret;
@@ -2496,6 +2526,16 @@ glusterd_op_stop_volume (gd1_mgmt_stage_op_req *req)
 
         ret = glusterd_volume_compute_cksum (volinfo);
 
+        if (glusterd_are_all_volumes_stopped ()) {
+                if (glusterd_is_nfs_started ()) {
+                        ret = glusterd_nfs_server_stop ();
+                        if (ret)
+                                goto out;
+                }
+        } else {
+                ret = glusterd_check_generate_start_nfs (volinfo);
+        }
+
 out:
         if (flags & GF_CLI_FLAG_OP_FORCE)
                 ret = 0;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 0861d0219..1c456afc6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1461,3 +1461,192 @@ out:
 
         return ret;
 }
+
+int
+glusterd_file_copy (int out, int in)
+{
+        int     read_size = 0;
+        char    buffer[16 * 1024];
+        int     ret = -1;
+
+        if (out <= 0 || in < 0) {
+                gf_log ("", GF_LOG_ERROR, "Invalid File descriptors");
+                goto out;
+        }
+
+        while (1) {
+                read_size = read(in, buffer, sizeof(buffer));
+
+                if (read_size == 0) {
+                        ret = 0;
+                        break;              /* end of file */
+                }
+
+                if (read_size < 0) {
+                        ret = -1;
+                        break; /*error reading file); */
+                }
+                write (out, buffer, (unsigned int) read_size);
+        }
+out:
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_nfs_started ()
+{
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        char                    pidfile[PATH_MAX] = {0,};
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        priv = this->private;
+
+        GLUSTERD_GET_NFS_PIDFILE(pidfile);
+        ret = access (pidfile, F_OK);
+
+        if (ret == 0)
+                return _gf_true;
+        else
+                return _gf_false;
+}
+
+int32_t
+glusterd_nfs_server_start ()
+{
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        char                    pidfile[PATH_MAX] = {0,};
+        char                    *volfile = NULL;
+        char                    path[PATH_MAX] = {0,};
+        char                    cmd_str[8192] = {0,};
+        char                    rundir[PATH_MAX] = {0,};
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        priv = this->private;
+
+        GLUSTERD_GET_NFS_DIR(path, priv);
+        snprintf (rundir, PATH_MAX, "%s/run", path);
+        ret = mkdir (rundir, 0777);
+
+        if ((ret == -1) && (EEXIST != errno)) {
+                gf_log ("", GF_LOG_ERROR, "Unable to create rundir %s",
+                        rundir);
+                goto out;
+        }
+
+        GLUSTERD_GET_NFS_PIDFILE(pidfile);
+        volfile = glusterd_get_nfs_filepath ();
+        if (!volfile) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = access (volfile, F_OK);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Nfs Volfile %s is not present",
+                        volfile);
+                goto out;
+        }
+
+        snprintf (cmd_str, 8192,
+                  "%s/sbin/glusterfs  -f %s -p %s",
+                  GFS_PREFIX, volfile, pidfile);
+        ret = gf_system (cmd_str);
+
+out:
+        if (volfile)
+                GF_FREE(volfile);
+        return ret;
+}
+
+int32_t
+glusterd_nfs_server_stop ()
+{
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        char                    pidfile[PATH_MAX] = {0,};
+        char                    path[PATH_MAX] = {0,};
+        pid_t                   pid = -1;
+        FILE                    *file = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        priv = this->private;
+
+        GLUSTERD_GET_NFS_DIR(path, priv);
+        GLUSTERD_GET_NFS_PIDFILE(pidfile);
+
+        file = fopen (pidfile, "r+");
+
+        if (!file) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
+                                pidfile);
+                ret = -1;
+                goto out;
+        }
+
+        ret = fscanf (file, "%d", &pid);
+        if (ret <= 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read pidfile: %s",
+                                pidfile);
+                ret = -1;
+                goto out;
+        }
+        fclose (file);
+        file = NULL;
+
+        gf_log ("", GF_LOG_NORMAL, "Stopping glusterfs running in pid: %d",
+                pid);
+
+        ret = kill (pid, SIGTERM);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to kill pid %d", pid);
+                goto out;
+        }
+
+        ret = unlink (pidfile);
+
+        if (ret && (ENOENT != errno)) {
+                gf_log ("", GF_LOG_ERROR, "Unable to unlink pidfile: %s",
+                                pidfile);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (file)
+                fclose (file);
+        return ret;
+}
+
+gf_boolean_t
+glusterd_are_all_volumes_stopped ()
+{
+        glusterd_conf_t                         *priv = NULL;
+        xlator_t                                *this = NULL;
+        glusterd_volinfo_t                      *voliter = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (voliter->status == GLUSTERD_STATUS_STARTED)
+                        return _gf_false;
+        }
+
+        return _gf_true;
+
+}
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 6af21a490..183642811 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -127,4 +127,19 @@ glusterd_compare_friend_data (dict_t  *vols, int32_t *status);
 int
 glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo);
 
+gf_boolean_t
+glusterd_is_nfs_started ();
+
+int32_t
+glusterd_nfs_server_start ();
+
+int32_t
+glusterd_nfs_server_stop ();
+
+int
+glusterd_file_copy (int out, int in);
+
+gf_boolean_t
+glusterd_are_all_volumes_stopped ();
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index d963519be..3380463d3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -36,6 +36,7 @@
 #include "cli1.h"
 #include "glusterd-mem-types.h"
 #include "glusterd-volgen.h"
+#include "glusterd-utils.h"
 
 static int
 set_xlator_option (dict_t *dict, char *key,
@@ -645,6 +646,31 @@ out:
 }
 
 static int
+__write_access_control_xlator (FILE *file, dict_t *dict,
+                               char *subvolume)
+{
+        char  *volname       = NULL;
+        int    ret           = -1;
+
+        const char *ac_str = "volume %s-access-control\n"
+                             "type features/access-control\n"
+                             "subvolumes %s\n"
+                             "end-volume\n";
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                goto out;
+        }
+
+
+        fprintf (file, ac_str, volname, subvolume);
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
 __write_server_xlator (FILE *file, dict_t *dict,
                        char *subvolume)
 {
@@ -1300,6 +1326,15 @@ generate_server_volfile (glusterd_brickinfo_t *brickinfo,
 
         VOLGEN_GENERATE_VOLNAME (subvol, volname, "posix");
 
+        ret = __write_access_control_xlator (file, dict, subvol);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "Could not write xlator");
+                goto out;
+        }
+
+        VOLGEN_GENERATE_VOLNAME (subvol, volname, "access-control");
+
         ret = __write_locks_xlator (file, dict, subvol);
         if (ret) {
                 gf_log ("", GF_LOG_DEBUG,
@@ -1584,8 +1619,31 @@ out:
         return ret;
 }
 
+char *
+glusterd_get_nfs_filepath ()
+{
+        char  path[PATH_MAX] = {0,};
+        char *ret            = NULL;
+        char *filepath       = NULL;
+
+        filepath = GF_CALLOC (1, PATH_MAX, gf_common_mt_char);
+        if (!filepath) {
+                gf_log ("", GF_LOG_ERROR, "Unable to allocate nfs file path");
+                goto out;
+        }
+
+        VOLGEN_GET_NFS_DIR (path);
+
+        snprintf (filepath, PATH_MAX, "%s/nfs-server.vol", path);
+
+        ret = filepath;
+out:
+        return ret;
+}
+
+
 static char *
-get_client_filename (glusterd_volinfo_t *volinfo)
+get_client_filepath (glusterd_volinfo_t *volinfo)
 {
         char  path[PATH_MAX] = {0,};
         char *ret            = NULL;
@@ -1644,12 +1702,161 @@ out:
 }
 
 static int
+glusterfsd_write_nfs_xlator (int fd, char *subvols)
+{
+        char    buffer[1024] = {0,};
+        if (fd <= 0)
+                return -1;
+        const char *nfs_str = "volume nfs-server\n"
+                              "type nfs/server\n"
+                              "subvolumes %s\n"
+                              "option rpc­auth.addr.allow *\n"
+                              "end-volume\n";
+        snprintf (buffer, sizeof(buffer), nfs_str, subvols);
+        write (fd, buffer, (unsigned int) strlen(buffer));
+        return 0;
+}
+
+int
+volgen_generate_nfs_volfile (glusterd_volinfo_t *volinfo)
+{
+        char                 *nfs_filepath      = NULL;
+        char                 *fuse_filepath     = NULL;
+        int                  nfs_fd             = -1;
+        int                  fuse_fd            = -1;
+        int                  ret                = -1;
+        char                 nfs_orig_path[PATH_MAX] = {0,};
+        char                 *pad               = NULL;
+        char                 *nfs_subvols       = NULL;
+        char                 fuse_subvols[2048] = {0,};
+        char                 *fuse_top_xlator = "stat-prefetch";
+        int                  subvol_len = 0;
+        glusterd_volinfo_t   *voliter = NULL;
+        glusterd_conf_t                         *priv = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        if (!volinfo) {
+                gf_log ("", GF_LOG_ERROR, "Invalid Volume info");
+                goto out;
+        }
+
+        nfs_filepath = glusterd_get_nfs_filepath (volinfo);
+        if (!nfs_filepath)
+                goto out;
+
+        strncat (nfs_filepath, ".tmp", PATH_MAX);
+        nfs_fd = open (nfs_filepath, O_WRONLY|O_TRUNC|O_CREAT, 0666);
+        if (nfs_fd < 0) {
+                gf_log ("", GF_LOG_ERROR, "Could not open file: %s",
+                        nfs_filepath);
+                goto out;
+        }
+
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (voliter->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+                else
+                        subvol_len += (strlen (voliter->volname) +
+                                       strlen (fuse_top_xlator) + 2); //- + ' '
+        }
+
+        if (subvol_len == 0) {
+                gf_log ("", GF_LOG_ERROR, "No volumes started");
+                ret = -1;
+                goto out;
+        } else {
+                subvol_len++; //null character
+                nfs_subvols = GF_CALLOC (subvol_len, sizeof(*nfs_subvols),
+                                         gf_common_mt_char);
+                if (!nfs_subvols) {
+                        gf_log ("", GF_LOG_ERROR, "Memory not available");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        voliter = NULL;
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (voliter->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+
+                gf_log ("", GF_LOG_DEBUG,
+                        "adding fuse info of - %s", voliter->volname);
+
+                snprintf (fuse_subvols, sizeof(fuse_subvols), " %s-%s",
+                          voliter->volname, fuse_top_xlator);
+                fuse_filepath = get_client_filepath (voliter);
+                if (!fuse_filepath) {
+                        ret = -1;
+                        goto out;
+                }
+
+                fuse_fd = open (fuse_filepath, O_RDONLY);
+                if (fuse_fd < 0) {
+                        gf_log ("", GF_LOG_ERROR, "Could not open file: %s",
+                                fuse_filepath);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_file_copy (nfs_fd, fuse_fd);
+                if (ret)
+                        goto out;
+                GF_FREE (fuse_filepath);
+                fuse_filepath = NULL;
+                close (fuse_fd);
+                fuse_fd = -1;
+                if (subvol_len > strlen (fuse_subvols)) {
+                        strncat (nfs_subvols, fuse_subvols, subvol_len - 1);
+                        subvol_len -= strlen (fuse_subvols);
+                } else {
+                        ret = -1;
+                        gf_log ("", GF_LOG_ERROR, "Too many subvolumes");
+                        goto out;
+                }
+        }
+
+        ret = glusterfsd_write_nfs_xlator (nfs_fd, nfs_subvols);
+        if (ret)
+                goto out;
+
+        strncpy (nfs_orig_path, nfs_filepath, PATH_MAX);
+        pad = strrchr (nfs_orig_path, '.');
+        if (!pad) {
+                gf_log ("", GF_LOG_ERROR, "Failed to find the pad in nfs pat");
+                ret = -1;
+                goto out;
+        }
+        *pad = '\0';
+        ret = rename (nfs_filepath, nfs_orig_path);
+out:
+        if (ret && nfs_filepath)
+                unlink (nfs_filepath);
+        if (fuse_filepath)
+                GF_FREE (fuse_filepath);
+        if (nfs_filepath)
+                GF_FREE (nfs_filepath);
+        if (nfs_subvols)
+                GF_FREE (nfs_subvols);
+        if (fuse_fd > 0)
+                close (fuse_fd);
+        if (nfs_fd > 0)
+                close (nfs_fd);
+        return ret;
+}
+
+static int
 generate_client_volfiles (glusterd_volinfo_t *volinfo)
 {
         char                 *filename    = NULL;
         int                   ret         = -1;
 
-        filename = get_client_filename (volinfo);
+        filename = get_client_filepath (volinfo);
         if (!filename) {
                 gf_log ("", GF_LOG_ERROR,
                         "Out of memory");
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 0ec8cc769..bee572646 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -37,6 +37,12 @@
 #include "cli1.h"
 #include "glusterd-mem-types.h"
 
+#define VOLGEN_GET_NFS_DIR(path)                                      \
+        do {                                                            \
+                glusterd_conf_t *priv = THIS->private;                  \
+                snprintf (path, PATH_MAX, "%s/nfs", priv->workdir);\
+        } while (0);                                                    \
+
 #define VOLGEN_GET_VOLUME_DIR(path, volinfo)                            \
         do {                                                            \
                 glusterd_conf_t *priv = THIS->private;                  \
@@ -130,4 +136,9 @@ glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
 int32_t
 glusterd_default_xlator_options (glusterd_volinfo_t *volinfo);
 
+char *
+glusterd_get_nfs_filepath ();
+
+int
+volgen_generate_nfs_volfile (glusterd_volinfo_t *volinfo);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 7653f6900..69ea9ed2c 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -313,6 +313,15 @@ init (xlator_t *this)
                 exit (1);
         }
 
+        snprintf (voldir, PATH_MAX, "%s/nfs", dirname);
+        ret = mkdir (voldir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create nfs directory %s"
+                        " ,errno = %d", voldir, errno);
+                exit (1);
+        }
+
         rpc = rpcsvc_init (this->ctx, this->options);
         if (rpc == NULL) {
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index df3bc9ee1..b7027b97c 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -175,6 +175,11 @@ enum glusterd_vol_comp_status_ {
 
 typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
 
+#define GLUSTERD_GET_NFS_DIR(path, priv)                                \
+        do {                                                            \
+                snprintf (path, PATH_MAX, "%s/nfs", priv->workdir);\
+        } while (0);                                                    \
+
 #define GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv) \
         snprintf (path, PATH_MAX, "%s/vols/%s", priv->workdir,\
                   volinfo->volname);
@@ -184,6 +189,10 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                   GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname, \
                   GLUSTERD_BRICK_INFO_DIR);
 
+#define GLUSTERD_GET_NFS_PIDFILE(pidfile)                               \
+                snprintf (pidfile, PATH_MAX, "%s/nfs/run/nfs.pid", \
+                          priv->workdir);                               \
+
 #define GLUSTERD_REMOVE_SLASH_FROM_PATH(path,string) do {               \
                 int i = 0;                                              \
                 for (i = 1; i < strlen (path); i++) {                   \