glusterfsd: Round robin DNS should not be relied upon with

config service availability for clients.

Backupvolfile server as it stands is slow and prone to errors
with mount script and its combination with RRDNS. Instead in
theory it should use all the available nodes in 'trusted pool'
by default (Right now we don't have a mechanism in place for
this)

Nevertheless this patch provides a scenario where a list of
volfile-server can be provided on command as shown below

-----------------------------------------------------------------
$ glusterfs -s server1 .. -s serverN --volfile-id=<volname> \
      <mount_point>
-----------------------------------------------------------------
                   OR
-----------------------------------------------------------------
$ mount -t glusterfs -obackup-volfile-servers=<server2>: \
      <server3>:...:<serverN> <server1>:/<volname> <mount_point>
-----------------------------------------------------------------

Here ':' is used as a separator for mount script parsing

Now these will be remembered and recursively attempted for
fetching vol-file until exhausted. This would ensure that the
clients get 'volume' configs in a consistent manner avoiding the
need to poll through RRDNS.

Change-Id: If808bb8a52e6034c61574cdae3ac4e7e83513a40
BUG: 986429
Signed-off-by: Harshavardhana <harsha@harshavardhana.net>
Reviewed-on: http://review.gluster.org/5400
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

3 files changed, 96 insertions, 36 deletions

diff --git a/glusterfsd/src/glusterfsd-mem-types.h b/glusterfsd/src/glusterfsd-mem-types.h
index 83f97b5d32a..7135c0ada9e 100644
--- a/glusterfsd/src/glusterfsd-mem-types.h
+++ b/glusterfsd/src/glusterfsd-mem-types.h
@@ -17,6 +17,7 @@
 enum gfd_mem_types_ {
         gfd_mt_xlator_list_t = GF_MEM_TYPE_START,
         gfd_mt_xlator_t,
+        gfd_mt_server_cmdline_t,
         gfd_mt_xlator_cmdline_option_t,
         gfd_mt_char,
         gfd_mt_call_pool_t,
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 4b9b82fff86..e40d19b0888 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1821,30 +1821,52 @@ static int
 mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                  void *data)
 {
-        xlator_t        *this = NULL;
-        cmd_args_t      *cmd_args = NULL;
-        glusterfs_ctx_t *ctx = NULL;
+        xlator_t         *this = NULL;
+        glusterfs_ctx_t  *ctx = NULL;
         int              ret = 0;
-        int need_term = 0;
-        int emval = 0;
+        server_cmdline_t *server = NULL;
+        rpc_transport_t  *rpc_trans = NULL;
+        int              need_term = 0;
+        int              emval = 0;
 
         this = mydata;
+        rpc_trans = rpc->conn.trans;
         ctx = this->ctx;
-        cmd_args = &ctx->cmd_args;
+
         switch (event) {
         case RPC_CLNT_DISCONNECT:
                 if (!ctx->active) {
-                        cmd_args->max_connect_attempts--;
                         gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-                                "failed to connect with remote-host: %s",
+                                "failed to connect with remote-host: %s (%s)",
+                                ctx->cmd_args.volfile_server,
                                 strerror (errno));
-                        gf_log ("glusterfsd-mgmt", GF_LOG_INFO,
-                                "%d connect attempts left",
-                                cmd_args->max_connect_attempts);
-                        if (0 >= cmd_args->max_connect_attempts) {
+                        server = ctx->cmd_args.curr_server;
+                        if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                                need_term = 1;
+                                emval = ENOTCONN;
+                                gf_log("glusterfsd-mgmt", GF_LOG_INFO,
+                                       "Exhausted all volfile servers");
+                                break;
+                        }
+                        server = list_entry (server->list.next, typeof(*server),
+                                             list);
+                        ctx->cmd_args.curr_server = server;
+                        ctx->cmd_args.volfile_server = server->volfile_server;
+
+                        ret = dict_set_str (rpc_trans->options,
+                                            "remote-host",
+                                            server->volfile_server);
+                        if (ret != 0) {
+                                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
+                                        "failed to set remote-host: %s",
+                                        server->volfile_server);
                                 need_term = 1;
                                 emval = ENOTCONN;
+                                break;
                         }
+                        gf_log ("glusterfsd-mgmt", GF_LOG_INFO,
+                                "connecting to next volfile server %s",
+                                server->volfile_server);
                 }
                 break;
         case RPC_CLNT_CONNECT:
@@ -2048,7 +2070,8 @@ glusterfs_mgmt_init (glusterfs_ctx_t *ctx)
 
         ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "failed to register notify function");
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to register notify function");
                 goto out;
         }
 
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 7b2ccd6eb1a..e8ebdae1498 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -98,10 +98,6 @@ static struct argp_option gf_options[] = {
         {"volfile-server", ARGP_VOLFILE_SERVER_KEY, "SERVER", 0,
          "Server to get the volume file from.  This option overrides "
          "--volfile option"},
-        {"volfile-max-fetch-attempts", ARGP_VOLFILE_MAX_FETCH_ATTEMPTS,
-         "MAX-ATTEMPTS", 0, "Maximum number of connect attempts to server. "
-         "This option should be provided with --volfile-server option"
-         "[default: 1]"},
         {"volfile", ARGP_VOLUME_FILE_KEY, "VOLFILE", 0,
          "File to use as VOLUME_FILE"},
         {"spec-file", ARGP_VOLUME_FILE_KEY, "VOLFILE", OPTION_HIDDEN,
@@ -582,7 +578,58 @@ get_volfp (glusterfs_ctx_t *ctx)
 }
 
 static int
-gf_remember_xlator_option (struct list_head *options, char *arg)
+gf_remember_backup_volfile_server (char *arg)
+{
+        glusterfs_ctx_t         *ctx = NULL;
+        cmd_args_t              *cmd_args = NULL;
+        int                      ret = -1;
+        server_cmdline_t        *server = NULL;
+
+        ctx = glusterfsd_ctx;
+        if (!ctx)
+                goto out;
+        cmd_args = &ctx->cmd_args;
+
+        if(!cmd_args)
+                goto out;
+
+        server = GF_CALLOC (1, sizeof (server_cmdline_t),
+                            gfd_mt_server_cmdline_t);
+        if (!server)
+                goto out;
+
+        INIT_LIST_HEAD(&server->list);
+
+        server->volfile_server = gf_strdup(arg);
+
+        if (!cmd_args->volfile_server) {
+                cmd_args->volfile_server = server->volfile_server;
+                cmd_args->curr_server = server;
+        }
+
+        if (!server->volfile_server) {
+                gf_log ("", GF_LOG_WARNING,
+                        "xlator option %s is invalid", arg);
+                goto out;
+        }
+
+        list_add_tail (&server->list, &cmd_args->volfile_servers);
+
+        ret = 0;
+out:
+        if (ret == -1) {
+                if (server) {
+                        GF_FREE (server->volfile_server);
+                        GF_FREE (server);
+                }
+        }
+
+        return ret;
+
+}
+
+static int
+gf_remember_xlator_option (char *arg)
 {
         glusterfs_ctx_t         *ctx = NULL;
         cmd_args_t              *cmd_args  = NULL;
@@ -673,19 +720,8 @@ parse_opts (int key, char *arg, struct argp_state *state)
 
         switch (key) {
         case ARGP_VOLFILE_SERVER_KEY:
-                cmd_args->volfile_server = gf_strdup (arg);
-                break;
-
-        case ARGP_VOLFILE_MAX_FETCH_ATTEMPTS:
-                n = 0;
+                gf_remember_backup_volfile_server (arg);
 
-                if (gf_string2uint_base10 (arg, &n) == 0) {
-                        cmd_args->max_connect_attempts = n;
-                        break;
-                }
-
-                argp_failure (state, -1, 0,
-                              "Invalid limit on connect attempts %s", arg);
                 break;
 
         case ARGP_READ_ONLY_KEY:
@@ -694,14 +730,12 @@ parse_opts (int key, char *arg, struct argp_state *state)
 
         case ARGP_ACL_KEY:
                 cmd_args->acl = 1;
-		gf_remember_xlator_option (&cmd_args->xlator_options,
-					   "*-md-cache.cache-posix-acl=true");
+                gf_remember_xlator_option ("*-md-cache.cache-posix-acl=true");
                 break;
 
         case ARGP_SELINUX_KEY:
                 cmd_args->selinux = 1;
-		gf_remember_xlator_option (&cmd_args->xlator_options,
-					   "*-md-cache.cache-selinux=true");
+                gf_remember_xlator_option ("*-md-cache.cache-selinux=true");
                 break;
 
         case ARGP_AUX_GFID_MOUNT_KEY:
@@ -906,8 +940,9 @@ parse_opts (int key, char *arg, struct argp_state *state)
                 break;
 
         case ARGP_XLATOR_OPTION_KEY:
-                if (gf_remember_xlator_option (&cmd_args->xlator_options, arg))
-                        argp_failure (state, -1, 0, "invalid xlator option  %s", arg);
+                if (gf_remember_xlator_option (arg))
+                        argp_failure (state, -1, 0, "invalid xlator option  %s",
+                                      arg);
 
                 break;
 
@@ -1297,6 +1332,7 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
         cmd_args->fuse_entry_timeout = -1;
 
         INIT_LIST_HEAD (&cmd_args->xlator_options);
+        INIT_LIST_HEAD (&cmd_args->volfile_servers);
 
         lim.rlim_cur = RLIM_INFINITY;
         lim.rlim_max = RLIM_INFINITY;