Fetch backup volfile servers from glusterd2

Clients will request for a list of volfile servers from glusterd2 by
setting a (optional) flag in GETSPEC RPC call. glusterd2 will check for
the presence of this flag and accordingly return a list of glusterd2
servers in GETSPEC RPC reply. Currently, this list of servers returned
only contains servers which have bricks belonging to the volume.

See:
https://github.com/gluster/glusterd2/issues/382
https://github.com/gluster/glusterfs/issues/351

Updates #351
Change-Id: I0eee3d0bf25a87627e562380ef73063926a16b81
Signed-off-by: Prashanth Pai <ppai@redhat.com>

9 files changed, 269 insertions, 83 deletions

diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index ac306e40027..f709b54d49e 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -580,6 +580,8 @@ glfs_mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
 	FILE			*tmpfp = NULL;
 	int			 need_retry = 0;
 	struct glfs		*fs = NULL;
+        dict_t                  *dict = NULL;
+        char                    *servers_list = NULL;
 
 	frame = myframe;
 	ctx = frame->this->ctx;
@@ -617,6 +619,44 @@ glfs_mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
 		goto out;
 	}
 
+        if (!rsp.xdata.xdata_len) {
+                goto volfile;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                errno = ENOMEM;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.xdata.xdata_val, rsp.xdata.xdata_len,
+                                &dict);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "failed to unserialize xdata to dictionary");
+                goto out;
+        }
+        dict->extra_stdfree = rsp.xdata.xdata_val;
+
+        /* glusterd2 only */
+        ret = dict_get_str (dict, "servers-list", &servers_list);
+        if (ret) {
+                goto volfile;
+        }
+
+        gf_log (frame->this->name, GF_LOG_INFO,
+                "Received list of available volfile servers: %s",
+                servers_list);
+
+        ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed (%s) to process servers list: %s",
+                        strerror (errno), servers_list);
+        }
+
+volfile:
 	ret = 0;
 	size = rsp.op_ret;
 
@@ -676,8 +716,8 @@ out:
 	if (rsp.spec)
 		free (rsp.spec);
 
-        if (rsp.xdata.xdata_val)
-                free (rsp.xdata.xdata_val);
+        if (dict)
+                dict_unref (dict);
 
 	// Stop if server is running at an unsupported op-version
 	if (ENOTSUP == ret) {
@@ -752,6 +792,11 @@ glfs_volfile_fetch (struct glfs *fs)
                 goto out;
         }
 
+        /* Ask for a list of volfile (glusterd2 only) servers */
+        if (GF_CLIENT_PROCESS == ctx->process_mode) {
+                req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+        }
+
         ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
                                            &req.xdata.xdata_len);
         if (ret < 0) {
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 2a7ae2f3986..6724534fb31 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -412,9 +412,9 @@ pub_glfs_set_volfile_server (struct glfs *fs, const char *transport,
                              const char *host, int port)
 {
         cmd_args_t            *cmd_args = NULL;
-        server_cmdline_t      *server = NULL;
-        server_cmdline_t      *tmp = NULL;
         int                    ret = -1;
+        char                  *server_host = NULL;
+        char                  *server_transport = NULL;
 
         if (!fs || !host) {
                 errno = EINVAL;
@@ -425,21 +425,10 @@ pub_glfs_set_volfile_server (struct glfs *fs, const char *transport,
         __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
 
         cmd_args = &fs->ctx->cmd_args;
-
         cmd_args->max_connect_attempts = 1;
 
-        server = GF_CALLOC (1, sizeof (server_cmdline_t),
-                            glfs_mt_server_cmdline_t);
-
-        if (!server) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&server->list);
-
-        server->volfile_server = gf_strdup (host);
-        if (!server->volfile_server) {
+        server_host = gf_strdup (host);
+        if (!server_host) {
                 errno = ENOMEM;
                 goto out;
         }
@@ -447,9 +436,10 @@ pub_glfs_set_volfile_server (struct glfs *fs, const char *transport,
         if (transport) {
                 /* volfile fetch support over tcp|unix only */
                 if (!strcmp(transport, "tcp") || !strcmp(transport, "unix")) {
-                        server->transport = gf_strdup (transport);
+                        server_transport = gf_strdup (transport);
                 } else if (!strcmp(transport, "rdma")) {
-                        server->transport = gf_strdup ("tcp");
+                        server_transport =
+                                gf_strdup (GF_DEFAULT_VOLFILE_TRANSPORT);
                         gf_msg ("glfs", GF_LOG_WARNING, EINVAL,
                                 API_MSG_INVALID_ENTRY,
                                 "transport RDMA is deprecated, "
@@ -460,55 +450,39 @@ pub_glfs_set_volfile_server (struct glfs *fs, const char *transport,
                                 "transport %s is not supported, "
                                 "possible values tcp|unix",
                                 transport);
-                        ret = -1;
                         goto out;
                 }
         } else {
-                server->transport = gf_strdup (GF_DEFAULT_VOLFILE_TRANSPORT);
+                server_transport = gf_strdup (GF_DEFAULT_VOLFILE_TRANSPORT);
         }
 
-        if (!server->transport) {
+        if (!server_transport) {
                 errno = ENOMEM;
                 goto out;
         }
 
-        if (strcmp(server->transport, "unix")) {
-                if (port) {
-                        server->port = port;
-                } else {
-                        server->port = GF_DEFAULT_BASE_PORT;
-                }
-        } else {
-                server->port = 0;
+        if (!port) {
+                port = GF_DEFAULT_BASE_PORT;
         }
 
-        if (!cmd_args->volfile_server) {
-                cmd_args->volfile_server = server->volfile_server;
-                cmd_args->volfile_server_transport = server->transport;
-                cmd_args->volfile_server_port = server->port;
-                cmd_args->curr_server = server;
+        if (!strcmp(server_transport, "unix")) {
+                port = 0;
         }
 
-        list_for_each_entry(tmp, &cmd_args->volfile_servers, list) {
-                if ((!strcmp(tmp->volfile_server, server->volfile_server) &&
-                     !strcmp(tmp->transport, server->transport) &&
-                     (tmp->port == server->port))) {
-                        errno = EEXIST;
-                        ret = -1;
-                        goto out;
-                }
+        ret = gf_set_volfile_server_common(cmd_args, server_host,
+                                           server_transport, port);
+        if (ret) {
+                gf_log ("glfs", GF_LOG_ERROR,
+                        "failed to set volfile server: %s", strerror (errno));
         }
 
-        list_add_tail (&server->list, &cmd_args->volfile_servers);
-
-        ret = 0;
 out:
-        if (ret == -1) {
-                if (server) {
-                        GF_FREE (server->volfile_server);
-                        GF_FREE (server->transport);
-                        GF_FREE (server);
-                }
+        if (server_host) {
+                GF_FREE (server_host);
+        }
+
+        if (server_transport) {
+                GF_FREE (server_transport);
         }
 
         __GLFS_EXIT_FS;
@@ -823,6 +797,7 @@ pub_glfs_new (const char *volname)
                 goto out;
 
         fs->ctx = ctx;
+        fs->ctx->process_mode = GF_CLIENT_PROCESS;
 
         ret = glfs_set_logging (fs, "/dev/null", 0);
         if (ret)
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 4c4d98dad3e..5d0c83fd1a6 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1917,6 +1917,8 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         gf_volfile_t            *volfile_obj = NULL;
         gf_volfile_t            *volfile_tmp = NULL;
         char                     sha256_hash[SHA256_DIGEST_LENGTH] = {0, };
+        dict_t                  *dict = NULL;
+        char                    *servers_list = NULL;
 
         frame = myframe;
         ctx = frame->this->ctx;
@@ -1940,6 +1942,44 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        if (!rsp.xdata.xdata_len) {
+                goto volfile;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                errno = ENOMEM;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.xdata.xdata_val, rsp.xdata.xdata_len,
+                                &dict);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "failed to unserialize xdata to dictionary");
+                goto out;
+        }
+        dict->extra_stdfree = rsp.xdata.xdata_val;
+
+        /* glusterd2 only */
+        ret = dict_get_str (dict, "servers-list", &servers_list);
+        if (ret) {
+                goto volfile;
+        }
+
+        gf_log (frame->this->name, GF_LOG_INFO,
+                "Received list of available volfile servers: %s",
+                servers_list);
+
+        ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed (%s) to process servers list: %s",
+                        strerror (errno), servers_list);
+        }
+
+volfile:
         ret = 0;
         size = rsp.op_ret;
 
@@ -2058,6 +2098,9 @@ out:
 
         free (rsp.spec);
 
+        if (dict)
+                dict_unref (dict);
+
         // Stop if server is running at an unsupported op-version
         if (ENOTSUP == ret) {
                 gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
@@ -2133,6 +2176,11 @@ glusterfs_volfile_fetch_one (glusterfs_ctx_t *ctx, char *volfile_id)
                 goto out;
         }
 
+        /* Ask for a list of volfile (glusterd2 only) servers */
+        if (GF_CLIENT_PROCESS == ctx->process_mode) {
+                req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+        }
+
         if (cmd_args->brick_name) {
                 ret = dict_set_dynstr_with_alloc (dict, "brick_name",
                                                   cmd_args->brick_name);
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 3bfd0f12384..dfaf9c09c6d 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -718,7 +718,6 @@ gf_remember_backup_volfile_server (char *arg)
         glusterfs_ctx_t         *ctx = NULL;
         cmd_args_t              *cmd_args = NULL;
         int                      ret = -1;
-        server_cmdline_t        *server = NULL;
 
         ctx = glusterfsd_ctx;
         if (!ctx)
@@ -728,39 +727,15 @@ gf_remember_backup_volfile_server (char *arg)
         if(!cmd_args)
                 goto out;
 
-        server = GF_CALLOC (1, sizeof (server_cmdline_t),
-                            gfd_mt_server_cmdline_t);
-        if (!server)
-                goto out;
-
-        INIT_LIST_HEAD(&server->list);
-
-        server->volfile_server = gf_strdup(arg);
-
-        if (!cmd_args->volfile_server) {
-                cmd_args->volfile_server = server->volfile_server;
-                cmd_args->curr_server = server;
-        }
-
-        if (!server->volfile_server) {
-                gf_msg ("glusterfsd", GF_LOG_WARNING, 0, glusterfsd_msg_10,
-                        "xlator option %s is invalid", arg);
-                goto out;
+        ret = gf_set_volfile_server_common(cmd_args, arg,
+                                           GF_DEFAULT_VOLFILE_TRANSPORT,
+                                           GF_DEFAULT_BASE_PORT);
+        if (ret) {
+                gf_log ("glusterfs", GF_LOG_ERROR,
+                        "failed to set volfile server: %s", strerror (errno));
         }
-
-        list_add_tail (&server->list, &cmd_args->volfile_servers);
-
-        ret = 0;
 out:
-        if (ret == -1) {
-                if (server) {
-                        GF_FREE (server->volfile_server);
-                        GF_FREE (server);
-                }
-        }
-
         return ret;
-
 }
 
 static int
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 5345924a41d..66461e3d440 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -3577,6 +3577,136 @@ out:
 
 }
 
+/*
+ * Processes list of volfile servers.
+ * Format: <host1>:<port1> <host2>:<port2>...
+ */
+int
+gf_process_getspec_servers_list(cmd_args_t *cmd_args,
+                                const char *servers_list) {
+        char        *tmp = NULL;
+        char        *address = NULL;
+        char        *host = NULL;
+        char        *last_colon = NULL;
+        char        *save_ptr = NULL;
+        int          port = 0;
+        int          ret = -1;
+
+        tmp = gf_strdup (servers_list);
+        if (!tmp) {
+                errno = ENOMEM;
+                goto out;
+        }
+
+        address = strtok_r (tmp, " ", &save_ptr);
+        if (!address) {
+                errno = EINVAL;
+                goto out;
+        }
+
+        while (1) {
+                last_colon = strrchr (address, ':');
+                if (!last_colon) {
+                        errno = EINVAL;
+                        ret = -1;
+                        break;
+                }
+                *last_colon = '\0';
+                host = address;
+                port = atoi (last_colon + 1);
+                if (port <= 0) {
+                        errno = EINVAL;
+                        ret = -1;
+                        break;
+                }
+                ret = gf_set_volfile_server_common (
+                        cmd_args, host, GF_DEFAULT_VOLFILE_TRANSPORT, port);
+                if (ret && errno != EEXIST) {
+                        break;
+                }
+                address = strtok_r (NULL, " ", &save_ptr);
+                if (!address) {
+                        errno = 0;
+                        ret = 0;
+                        break;
+                }
+        }
+
+out:
+        if (tmp) {
+                GF_FREE (tmp);
+        }
+
+        return ret;
+}
+
+int
+gf_set_volfile_server_common (cmd_args_t *cmd_args, const char *host,
+                              const char *transport, int port)
+{
+        server_cmdline_t      *server = NULL;
+        server_cmdline_t      *tmp = NULL;
+        int                    ret = -1;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, cmd_args, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, host, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, transport, out);
+
+        server = GF_CALLOC (1, sizeof (server_cmdline_t),
+                            gf_common_mt_server_cmdline_t);
+        if (!server) {
+                errno = ENOMEM;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&server->list);
+
+        server->volfile_server = gf_strdup (host);
+        if (!server->volfile_server) {
+                errno = ENOMEM;
+                goto out;
+        }
+
+        server->transport = gf_strdup (transport);
+        if (!server->transport) {
+                errno = ENOMEM;
+                goto out;
+        }
+
+        server->port = port;
+
+        if (!cmd_args->volfile_server) {
+                cmd_args->volfile_server = server->volfile_server;
+                cmd_args->volfile_server_transport = server->transport;
+                cmd_args->volfile_server_port = server->port;
+                cmd_args->curr_server = server;
+        }
+
+        list_for_each_entry(tmp, &cmd_args->volfile_servers, list) {
+                if ((!strcmp(tmp->volfile_server, server->volfile_server) &&
+                     !strcmp(tmp->transport, server->transport) &&
+                     (tmp->port == server->port))) {
+                        errno = EEXIST;
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        list_add_tail (&server->list, &cmd_args->volfile_servers);
+
+        ret = 0;
+out:
+        if (-1 == ret) {
+                if (server) {
+                        GF_FREE (server->volfile_server);
+                        GF_FREE (server->transport);
+                        GF_FREE (server);
+                }
+        }
+
+        return ret;
+}
+
 /* Sets log file path from user provided arguments */
 int
 gf_set_log_file_path (cmd_args_t *cmd_args, glusterfs_ctx_t *ctx)
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index c7f9fd58cc8..1cf517738c4 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -246,6 +246,11 @@ void gf_print_trace (int32_t signal, glusterfs_ctx_t *ctx);
 int  gf_set_log_file_path (cmd_args_t *cmd_args, glusterfs_ctx_t *ctx);
 int  gf_set_log_ident (cmd_args_t *cmd_args);
 
+int gf_process_getspec_servers_list(cmd_args_t *cmd_args,
+                                    const char *servers_list);
+int gf_set_volfile_server_common (cmd_args_t *cmd_args, const char *host,
+                                  const char *transport, int port);
+
 static inline void
 BIT_SET (unsigned char *array, unsigned int index)
 {
diff --git a/libglusterfs/src/libglusterfs.sym b/libglusterfs/src/libglusterfs.sym
index 39f03cbfa44..fa1e54d3663 100644
--- a/libglusterfs/src/libglusterfs.sym
+++ b/libglusterfs/src/libglusterfs.sym
@@ -632,6 +632,7 @@ gf_proc_dump_xlator_history
 gf_proc_dump_xlator_meminfo
 gf_proc_dump_xlator_private
 gf_proc_dump_xlator_profile
+gf_process_getspec_servers_list
 gf_process_reserved_ports
 __gf_realloc
 _gf_ref_get
@@ -646,6 +647,7 @@ gf_rsync_weak_checksum
 gf_set_log_file_path
 gf_set_log_ident
 gf_set_timestamp
+gf_set_volfile_server_common
 _gf_smsg
 gf_sock_union_equal_addr
 gf_store_handle_create_on_absence
diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h
index 64d0e90e1bd..85c06e9cbe5 100644
--- a/libglusterfs/src/mem-types.h
+++ b/libglusterfs/src/mem-types.h
@@ -176,6 +176,7 @@ enum gf_common_mem_types_ {
         gf_common_ping_local_t,
         gf_common_volfile_t,
         gf_common_mt_mgmt_v3_lock_timer_t,
+        gf_common_mt_server_cmdline_t,
         gf_common_mt_end
 };
 #endif
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index e922c890854..64723420445 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -325,6 +325,11 @@ enum gf_get_snapshot_info_type {
 };
 typedef enum gf_get_snapshot_info_type gf_get_snapshot_info_type;
 
+enum gf_getspec_flags_type {
+        GF_GETSPEC_FLAG_SERVERS_LIST = 1
+};
+typedef enum gf_getspec_flags_type gf_getspec_flags_type;
+
 #define GLUSTER_HNDSK_PROGRAM    14398633 /* Completely random */
 #define GLUSTER_HNDSK_VERSION    2   /* 0.0.2 */