glusterd: Bug fixes for IPv6 support

Problem:
Glusterd not working using ipv6 transport. The idea is with proper glusterd.vol configuration,
1. glusterd needs to listen on default port (240007) as IPv6 TCP listner.
2. Volume creation/deletion/mounting/add-bricks/delete-bricks/peer-probe
   needs to work using ipv6 addresses.
3. Bricks needs to listen on ipv6 addresses.
All the above functionality is needed to say that glusterd supports ipv6 transport and this is broken.

Fix:
When "option transport.address-family inet6" option is present in glusterd.vol
file, it is made sure that glusterd creates listeners using ipv6 sockets only and also the same information is saved
inside brick volume files used by glusterfsd brick process when they are starting.

Tests Run:
Regression tests using ./run-tests.sh
    IPv4: Ran manually till tests/basic/rpm.t .
    IPv6: (Need to add the above mentioned config and also add an entry for "hostname ::1" in /etc/hosts)
        Started failing at ./tests/basic/glusterd/arbiter-volume-probe.t and ran successfully till here

Unit Tests using Ipv6
    peer probe
    add-bricks
    remove-bricks
    create volume
    replace-bricks
    start volume
    stop volume
    delete volume

Change-Id: Iebc96e6cce748b5924ce5da17b0114600ec70a6e
BUG: 1117886
Signed-off-by: Nithin D <nithind1988@yahoo.in>
Reviewed-on: http://review.gluster.org/11988
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

13 files changed, 162 insertions, 25 deletions

diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index 593611612b8..4587e20df35 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -713,6 +713,7 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         rpc_transport_t  *rpc_trans = NULL;
 	struct glfs	 *fs = NULL;
 	int		 ret = 0;
+	struct dnscache6 *dnscache = NULL;
 
 	this = mydata;
         rpc_trans = rpc->conn.trans;
@@ -731,6 +732,17 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                                 "failed to connect with remote-host: %s (%s)",
                                 ctx->cmd_args.volfile_server,
                                 strerror (errno));
+
+                        if (!rpc->disabled) {
+                                /*
+                                 * Check if dnscache is exhausted for current server
+                                 * and continue until cache is exhausted
+                                 */
+                                dnscache = rpc_trans->dnscache;
+                                if (dnscache && dnscache->next) {
+                                        break;
+                                }
+                        }
                         server = ctx->cmd_args.curr_server;
                         if (server->list.next == &ctx->cmd_args.volfile_servers) {
                                 errno = ENOTCONN;
diff --git a/extras/glusterd.vol.in b/extras/glusterd.vol.in
index 316104c776a..957b277801c 100644
--- a/extras/glusterd.vol.in
+++ b/extras/glusterd.vol.in
@@ -7,5 +7,6 @@ volume management
     option transport.socket.read-fail-log off
     option ping-timeout 0
     option event-threads 1
+#   option transport.address-family inet6
 #   option base-port 49152
 end-volume
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index f7882723b1b..7af15eba92e 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1889,6 +1889,7 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         rpc_transport_t  *rpc_trans = NULL;
         int              need_term = 0;
         int              emval = 0;
+        struct dnscache6 *dnscache = NULL;
 
         this = mydata;
         rpc_trans = rpc->conn.trans;
@@ -1901,6 +1902,16 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                                 "failed to connect with remote-host: %s (%s)",
                                 ctx->cmd_args.volfile_server,
                                 strerror (errno));
+                        if (!rpc->disabled) {
+                                /*
+                                 * Check if dnscache is exhausted for current server
+                                 * and continue until cache is exhausted
+                                 */
+                                dnscache = rpc_trans->dnscache;
+                                if (dnscache && dnscache->next) {
+                                        break;
+                                }
+                        }
                         server = ctx->cmd_args.curr_server;
                         if (server->list.next == &ctx->cmd_args.volfile_servers) {
                                 need_term = 1;
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index cf5b524dc12..a1b67596c6e 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -55,11 +55,6 @@
 typedef int32_t (*rw_op_t)(int32_t fd, char *buf, int32_t size);
 typedef int32_t (*rwv_op_t)(int32_t fd, const struct iovec *buf, int32_t size);
 
-struct dnscache6 {
-        struct addrinfo *first;
-        struct addrinfo *next;
-};
-
 void
 md5_wrapper(const unsigned char *data, size_t len, char *md5)
 {
@@ -288,9 +283,6 @@ gf_resolve_ip6 (const char *hostname,
                 memset(&hints, 0, sizeof(hints));
                 hints.ai_family   = family;
                 hints.ai_socktype = SOCK_STREAM;
-#ifndef __NetBSD__
-                hints.ai_flags    = AI_ADDRCONFIG;
-#endif
 
                 ret = gf_asprintf (&port_str, "%d", port);
                 if (-1 == ret) {
@@ -2321,6 +2313,14 @@ valid_ipv6_address (char *address, int length, gf_boolean_t wildcard_acc)
 
         tmp = gf_strdup (address);
 
+        /* Check for '%' for link local addresses */
+        endptr = strchr(tmp, '%');
+        if (endptr) {
+                *endptr = '\0';
+                length = strlen(tmp);
+                endptr = NULL;
+        }
+
         /* Check for compressed form */
         if (length <= 0 || tmp[length - 1] == ':') {
                 ret = 0;
@@ -3247,9 +3247,18 @@ gf_is_local_addr (char *hostname)
         gf_boolean_t    found = _gf_false;
         char            *ip = NULL;
         xlator_t        *this = NULL;
+        struct          addrinfo hints;
 
         this = THIS;
-        ret = getaddrinfo (hostname, NULL, NULL, &result);
+
+        memset (&hints, 0, sizeof (hints));
+        /*
+         * Removing AI_ADDRCONFIG from default_hints
+         * for being able to use link local ipv6 addresses
+         */
+        hints.ai_family = AF_UNSPEC;
+
+        ret = getaddrinfo (hostname, NULL, &hints, &result);
 
         if (ret != 0) {
                 gf_msg (this->name, GF_LOG_ERROR, 0, LG_MSG_GETADDRINFO_FAILED,
@@ -3289,15 +3298,19 @@ gf_is_same_address (char *name1, char *name2)
         struct addrinfo         *q = NULL;
         gf_boolean_t            ret = _gf_false;
         int                     gai_err = 0;
+        struct                  addrinfo hints;
+
+        memset (&hints, 0, sizeof (hints));
+        hints.ai_family = AF_UNSPEC;
 
-        gai_err = getaddrinfo(name1,NULL,NULL,&addr1);
+        gai_err = getaddrinfo(name1, NULL, &hints, &addr1);
         if (gai_err != 0) {
                 gf_msg (name1, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED,
                         "error in getaddrinfo: %s\n", gai_strerror(gai_err));
                 goto out;
         }
 
-        gai_err = getaddrinfo(name2,NULL,NULL,&addr2);
+        gai_err = getaddrinfo(name2, NULL, &hints, &addr2);
         if (gai_err != 0) {
                 gf_msg (name2, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED,
                         "error in getaddrinfo: %s\n", gai_strerror(gai_err));
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index f5f4493e21b..bd1a491f1a9 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -176,6 +176,12 @@ struct dnscache_entry {
         time_t timestamp;
 };
 
+struct dnscache6 {
+        struct addrinfo *first;
+        struct addrinfo *next;
+};
+
+
 
 struct dnscache *gf_dnscache_init (time_t ttl);
 struct dnscache_entry *gf_dnscache_entry_init ();
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index 0f1351fe91a..ca5d51488f2 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -687,12 +687,6 @@ rpc_transport_inet_options_build (dict_t **options, const char *hostname,
                         "failed to set remote-port with %d", port);
                 goto out;
         }
-        ret = dict_set_str (dict, "transport.address-family", "inet");
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set addr-family with inet");
-                goto out;
-        }
 
         ret = dict_set_str (dict, "transport-type", "socket");
         if (ret) {
diff --git a/rpc/rpc-transport/socket/src/name.c b/rpc/rpc-transport/socket/src/name.c
index 650c5a747be..a8cf04d8ab7 100644
--- a/rpc/rpc-transport/socket/src/name.c
+++ b/rpc/rpc-transport/socket/src/name.c
@@ -149,9 +149,10 @@ client_fill_address_family (rpc_transport_t *this, sa_family_t *sa_family)
 
                 if (remote_host_data) {
                         gf_log (this->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be inet from (remote-host: %s)", data_to_str (remote_host_data));
-                        *sa_family = AF_INET;
+                                "address-family not specified, marking it as unspec "
+                                "for getaddrinfo to resolve from (remote-host: %s)",
+                                data_to_str(remote_host_data));
+                        *sa_family = AF_UNSPEC;
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "address-family not specified, guessing it "
@@ -395,7 +396,7 @@ af_inet_server_get_local_sockaddr (rpc_transport_t *this,
         memset (&hints, 0, sizeof (hints));
         hints.ai_family = addr->sa_family;
         hints.ai_socktype = SOCK_STREAM;
-        hints.ai_flags    = AI_PASSIVE | AI_ADDRCONFIG;
+        hints.ai_flags    = AI_PASSIVE;
 
         ret = getaddrinfo(listen_host, service, &hints, &res);
         if (ret != 0) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 3ecb4f064f1..093548de712 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -266,6 +266,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         xlator_t                                *this               = NULL;
         gf_boolean_t                             is_force           = _gf_false;
         gsync_status_param_t                     param              = {0,};
+        char                                    *c                  = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -417,8 +418,18 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         GD_MSG_NO_MEMORY, "Memory allocation failed");
                 goto out;
         }
-        host = strtok_r (dup_dstbrick, ":", &savetok);
-        path = strtok_r (NULL, ":", &savetok);
+
+        /*
+         * IPv4 address contains '.' and ipv6 addresses contains ':'
+         * So finding the last occurance of ':' to
+         * mark the start of brick path
+         */
+        c = strrchr(dup_dstbrick, ':');
+        if (c != NULL) {
+                c[0] = '\0';
+                host = dup_dstbrick;
+                path = c++;
+        }
 
         if (!host || !path) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index c808b07cbf2..cc599846bb9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -2210,6 +2210,7 @@ brick_graph_add_server (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char            key[1024] = {0};
         char            *ssl_user = NULL;
         char            *value = NULL;
+        char            *address_family_data = NULL;
 
         if (!graph || !volinfo || !set_dict || !brickinfo)
                 goto out;
@@ -2246,6 +2247,17 @@ brick_graph_add_server (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         RPC_SET_OPT(xl, SSL_DH_PARAM_OPT,   "ssl-dh-param",         return -1);
         RPC_SET_OPT(xl, SSL_EC_CURVE_OPT,   "ssl-ec-curve",         return -1);
 
+        if (dict_get_str (volinfo->dict, "transport.address-family",
+                &address_family_data) == 0) {
+                ret = xlator_set_option (xl, "transport.address-family",
+                                address_family_data);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_WARNING,
+                                        "failed to set transport.address-family");
+                        return -1;
+                }
+        }
+
         if (username) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "auth.login.%s.allow",
@@ -2293,6 +2305,7 @@ brick_graph_add_pump (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char            *password      = NULL;
         char            *ptranst       = NULL;
         char            *value         = NULL;
+        char            *address_family_data = NULL;
 
 
         if (!graph || !volinfo || !set_dict)
@@ -2354,6 +2367,18 @@ brick_graph_add_pump (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 if (ret)
                         return -1;
 
+                if (dict_get_str (volinfo->dict, "transport.address-family",
+                                        &address_family_data) == 0) {
+                        ret = xlator_set_option (rbxl,
+                                        "transport.address-family",
+                                        address_family_data);
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_WARNING,
+                                                "failed to set transport.address-family");
+                                return -1;
+                        }
+                }
+
                 xl = volgen_graph_add_nolink (graph, "cluster/pump", "%s-pump",
                                               volinfo->volname);
                 if (!xl)
@@ -2831,6 +2856,7 @@ volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char                    *ssl_str            = NULL;
         gf_boolean_t             ssl_bool           = _gf_false;
         char                    *value              = NULL;
+        char                    *address_family_data = NULL;
 
         GF_ASSERT (graph);
         GF_ASSERT (subvol);
@@ -2860,6 +2886,18 @@ volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 goto err;
 
+        if (dict_get_str (volinfo->dict, "transport.address-family",
+                                &address_family_data) == 0) {
+                ret = xlator_set_option (xl,
+                                "transport.address-family",
+                                address_family_data);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_WARNING,
+                                "failed to set transport.address-family");
+                        goto err;
+                }
+        }
+
         ret = dict_get_uint32 (set_dict, "trusted-client",
                                &client_type);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 4850ce64fee..f02ac80ccb0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -279,6 +279,7 @@ __glusterd_handle_create_volume (rpcsvc_request_t *req)
         xlator_t               *this        = NULL;
         char                   *free_ptr    = NULL;
         char                   *trans_type  = NULL;
+        char                   *address_family_str  = NULL;
         uuid_t                  volume_id   = {0,};
         uuid_t                  tmp_uuid    = {0};
         int32_t                 type        = 0;
@@ -359,6 +360,8 @@ __glusterd_handle_create_volume (rpcsvc_request_t *req)
                 goto out;
         }
 
+
+
         ret = dict_get_str (dict, "transport", &trans_type);
         if (ret) {
                 snprintf (err_str, sizeof (err_str), "Unable to get "
@@ -367,6 +370,30 @@ __glusterd_handle_create_volume (rpcsvc_request_t *req)
                         GD_MSG_DICT_GET_FAILED, "%s", err_str);
                 goto out;
         }
+
+        ret = dict_get_str (this->options, "transport.address-family",
+                        &address_family_str);
+
+        if (!ret) {
+                ret = dict_set_dynstr_with_alloc (dict,
+                                "transport.address-family",
+                                address_family_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to set transport.address-family");
+                        goto out;
+                }
+        } else if (!strcmp(trans_type, "tcp")) {
+                /* Setting default as inet for trans_type tcp */
+                ret = dict_set_dynstr_with_alloc (dict,
+                                "transport.address-family",
+                                "inet");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to set transport.address-family");
+                        goto out;
+                }
+        }
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
                 snprintf (err_str, sizeof (err_str), "Unable to get bricks for "
@@ -2084,6 +2111,7 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         char                  msg[1024] __attribute__((unused)) = {0, };
         char                 *brick_mount_dir = NULL;
         char                  key[PATH_MAX]   = "";
+        char                 *address_family_str = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -2356,6 +2384,20 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = dict_get_str (dict, "transport.address-family",
+                        &address_family_str);
+
+        if (!ret) {
+                ret = dict_set_dynstr_with_alloc(volinfo->dict,
+                                "transport.address-family", address_family_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set transport.address-family for %s",
+                                        volinfo->volname);
+                        goto out;
+                }
+        }
+
         gd_update_volume_op_versions (volinfo);
 
         volinfo->caps = caps;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index c0059d83cfe..dd9d159f0ca 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1636,6 +1636,12 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .option     = "!ssl-ec-curve",
           .op_version = GD_OP_VERSION_3_7_4,
         },
+        { .key         = "transport.address-family",
+          .voltype     = "protocol/server",
+          .option      = "!address-family",
+          .op_version  = GD_OP_VERSION_3_7_4,
+          .type        = NO_DOC,
+        },
 
         /* Performance xlators enable/disbable options */
         { .key         = "performance.write-behind",
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 32168df5a33..78e32826b0b 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -617,7 +617,8 @@ main ()
     fi
 
     [ -r "$volfile_loc" ] || {
-        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        # '%' included to support ipv6 link local addresses
+        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:%.\-]*\):.*/\1/p');
         volume_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
         [ -n "$volume_str" ] && {
             volume_id="$volume_str";
diff --git a/xlators/mount/fuse/utils/mount_glusterfs.in b/xlators/mount/fuse/utils/mount_glusterfs.in
index 50a301c26a8..ea34dabf0b6 100755
--- a/xlators/mount/fuse/utils/mount_glusterfs.in
+++ b/xlators/mount/fuse/utils/mount_glusterfs.in
@@ -494,7 +494,8 @@ main ()
     done
 
     [ -r "$volfile_loc" ] || {
-        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        # '%' included to support ipv6 link local addresses
+        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:%.\-]*\):.*/\1/p');
         volume_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
         [ -n "$volume_str" ] && {
             volume_id="$volume_str";