glusterd: Bug fixes for IPv6 support

Backport of http://review.gluster.org/#/c/11988/

Problem:
Glusterd not working using ipv6 transport. The idea is with proper glusterd.vol configuration,
1. glusterd needs to listen on default port (240007) as IPv6 TCP listner.
2. Volume creation/deletion/mounting/add-bricks/delete-bricks/peer-probe
needs to work using ipv6 addresses.
3. Bricks needs to listen on ipv6 addresses.
All the above functionality is needed to say that glusterd supports ipv6 transport and this is broken.

Fix:
When "option transport.address-family inet6" option is present in glusterd.vol
file, it is made sure that glusterd creates listeners using ipv6 sockets only and also the same information is saved
inside brick volume files used by glusterfsd brick process when they are starting.

Tests Run:
Regression tests using ./run-tests.sh
IPv4: Regression tests using ./run-tests.sh for release-3.7 branch verified by comparing with clean repo.
IPv6: (Need to add the above mentioned config and also add an entry for "hostname ::1" in /etc/hosts)
    Started failing at ./tests/basic/glusterd/arbiter-volume-probe.t and ran successfully till here

Change-Id: Idd7513aa2347ce0de2b1f68daeecce1b7a39a7af
BUG: 1310445
Signed-off-by: Nithin D <nithind1988@yahoo.in>
Reviewed-on: http://review.gluster.org/13787
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

13 files changed, 162 insertions, 25 deletions

diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index ba1f4599488..107820c2a6d 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -718,6 +718,7 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         rpc_transport_t  *rpc_trans = NULL;
 	struct glfs	 *fs = NULL;
 	int		 ret = 0;
+	struct dnscache6 *dnscache = NULL;
 
 	this = mydata;
         rpc_trans = rpc->conn.trans;
@@ -736,6 +737,17 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                                 "failed to connect with remote-host: %s (%s)",
                                 ctx->cmd_args.volfile_server,
                                 strerror (errno));
+
+                        if (!rpc->disabled) {
+                                /*
+                                 * Check if dnscache is exhausted for current server
+                                 * and continue until cache is exhausted
+                                 */
+                                dnscache = rpc_trans->dnscache;
+                                if (dnscache && dnscache->next) {
+                                        break;
+                                }
+                        }
                         server = ctx->cmd_args.curr_server;
                         if (server->list.next == &ctx->cmd_args.volfile_servers) {
                                 errno = ENOTCONN;
diff --git a/extras/glusterd.vol.in b/extras/glusterd.vol.in
index 316104c776a..957b277801c 100644
--- a/extras/glusterd.vol.in
+++ b/extras/glusterd.vol.in
@@ -7,5 +7,6 @@ volume management
     option transport.socket.read-fail-log off
     option ping-timeout 0
     option event-threads 1
+#   option transport.address-family inet6
 #   option base-port 49152
 end-volume
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index da801779079..08e8e15ae29 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1893,6 +1893,7 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         rpc_transport_t  *rpc_trans = NULL;
         int              need_term = 0;
         int              emval = 0;
+        struct dnscache6 *dnscache = NULL;
 
         this = mydata;
         rpc_trans = rpc->conn.trans;
@@ -1905,6 +1906,16 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                                 "failed to connect with remote-host: %s (%s)",
                                 ctx->cmd_args.volfile_server,
                                 strerror (errno));
+                        if (!rpc->disabled) {
+                                /*
+                                 * Check if dnscache is exhausted for current server
+                                 * and continue until cache is exhausted
+                                 */
+                                dnscache = rpc_trans->dnscache;
+                                if (dnscache && dnscache->next) {
+                                        break;
+                                }
+                        }
                         server = ctx->cmd_args.curr_server;
                         if (server->list.next == &ctx->cmd_args.volfile_servers) {
                                 need_term = 1;
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 81e01bf6108..e3293dc55ca 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -59,11 +59,6 @@
 typedef int32_t (*rw_op_t)(int32_t fd, char *buf, int32_t size);
 typedef int32_t (*rwv_op_t)(int32_t fd, const struct iovec *buf, int32_t size);
 
-struct dnscache6 {
-        struct addrinfo *first;
-        struct addrinfo *next;
-};
-
 void
 md5_wrapper(const unsigned char *data, size_t len, char *md5)
 {
@@ -292,9 +287,6 @@ gf_resolve_ip6 (const char *hostname,
                 memset(&hints, 0, sizeof(hints));
                 hints.ai_family   = family;
                 hints.ai_socktype = SOCK_STREAM;
-#ifndef __NetBSD__
-                hints.ai_flags    = AI_ADDRCONFIG;
-#endif
 
                 ret = gf_asprintf (&port_str, "%d", port);
                 if (-1 == ret) {
@@ -2193,6 +2185,14 @@ valid_ipv6_address (char *address, int length, gf_boolean_t wildcard_acc)
 
         tmp = gf_strdup (address);
 
+        /* Check for '%' for link local addresses */
+        endptr = strchr(tmp, '%');
+        if (endptr) {
+                *endptr = '\0';
+                length = strlen(tmp);
+                endptr = NULL;
+        }
+
         /* Check for compressed form */
         if (length <= 0 || tmp[length - 1] == ':') {
                 ret = 0;
@@ -3118,9 +3118,18 @@ gf_is_local_addr (char *hostname)
         gf_boolean_t    found = _gf_false;
         char            *ip = NULL;
         xlator_t        *this = NULL;
+        struct          addrinfo hints;
 
         this = THIS;
-        ret = getaddrinfo (hostname, NULL, NULL, &result);
+
+        memset (&hints, 0, sizeof (hints));
+        /*
+         * Removing AI_ADDRCONFIG from default_hints
+         * for being able to use link local ipv6 addresses
+         */
+        hints.ai_family = AF_UNSPEC;
+
+        ret = getaddrinfo (hostname, NULL, &hints, &result);
 
         if (ret != 0) {
                 gf_msg (this->name, GF_LOG_ERROR, 0, LG_MSG_GETADDRINFO_FAILED,
@@ -3160,15 +3169,19 @@ gf_is_same_address (char *name1, char *name2)
         struct addrinfo         *q = NULL;
         gf_boolean_t            ret = _gf_false;
         int                     gai_err = 0;
+        struct                  addrinfo hints;
+
+        memset (&hints, 0, sizeof (hints));
+        hints.ai_family = AF_UNSPEC;
 
-        gai_err = getaddrinfo(name1,NULL,NULL,&addr1);
+        gai_err = getaddrinfo(name1, NULL, &hints, &addr1);
         if (gai_err != 0) {
                 gf_msg (name1, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED,
                         "error in getaddrinfo: %s\n", gai_strerror(gai_err));
                 goto out;
         }
 
-        gai_err = getaddrinfo(name2,NULL,NULL,&addr2);
+        gai_err = getaddrinfo(name2, NULL, &hints, &addr2);
         if (gai_err != 0) {
                 gf_msg (name2, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED,
                         "error in getaddrinfo: %s\n", gai_strerror(gai_err));
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index e374285c71d..c152540f0ff 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -165,6 +165,12 @@ typedef struct dht_changelog_rename_info {
          char    buffer[1];
  } dht_changelog_rename_info_t;
 
+struct dnscache6 {
+        struct addrinfo *first;
+        struct addrinfo *next;
+};
+
+
 
 typedef int (*gf_cmp) (void *, void *);
 
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index 0904dbee716..432bf469934 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -692,12 +692,6 @@ rpc_transport_inet_options_build (dict_t **options, const char *hostname,
                         "failed to set remote-port with %d", port);
                 goto out;
         }
-        ret = dict_set_str (dict, "transport.address-family", "inet");
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set addr-family with inet");
-                goto out;
-        }
 
         ret = dict_set_str (dict, "transport-type", "socket");
         if (ret) {
diff --git a/rpc/rpc-transport/socket/src/name.c b/rpc/rpc-transport/socket/src/name.c
index 650c5a747be..a8cf04d8ab7 100644
--- a/rpc/rpc-transport/socket/src/name.c
+++ b/rpc/rpc-transport/socket/src/name.c
@@ -149,9 +149,10 @@ client_fill_address_family (rpc_transport_t *this, sa_family_t *sa_family)
 
                 if (remote_host_data) {
                         gf_log (this->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be inet from (remote-host: %s)", data_to_str (remote_host_data));
-                        *sa_family = AF_INET;
+                                "address-family not specified, marking it as unspec "
+                                "for getaddrinfo to resolve from (remote-host: %s)",
+                                data_to_str(remote_host_data));
+                        *sa_family = AF_UNSPEC;
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "address-family not specified, guessing it "
@@ -395,7 +396,7 @@ af_inet_server_get_local_sockaddr (rpc_transport_t *this,
         memset (&hints, 0, sizeof (hints));
         hints.ai_family = addr->sa_family;
         hints.ai_socktype = SOCK_STREAM;
-        hints.ai_flags    = AI_PASSIVE | AI_ADDRCONFIG;
+        hints.ai_flags    = AI_PASSIVE;
 
         ret = getaddrinfo(listen_host, service, &hints, &res);
         if (ret != 0) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index f1c3637270b..a3620bd33b3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -266,6 +266,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         xlator_t                                *this               = NULL;
         gf_boolean_t                             is_force           = _gf_false;
         gsync_status_param_t                     param              = {0,};
+        char                                    *c                  = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -417,8 +418,18 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         GD_MSG_NO_MEMORY, "Memory allocation failed");
                 goto out;
         }
-        host = strtok_r (dup_dstbrick, ":", &savetok);
-        path = strtok_r (NULL, ":", &savetok);
+
+        /*
+         * IPv4 address contains '.' and ipv6 addresses contains ':'
+         * So finding the last occurance of ':' to
+         * mark the start of brick path
+         */
+        c = strrchr(dup_dstbrick, ':');
+        if (c != NULL) {
+                c[0] = '\0';
+                host = dup_dstbrick;
+                path = c++;
+        }
 
         if (!host || !path) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 8a86ab53977..f1d9217be59 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -2086,6 +2086,7 @@ brick_graph_add_server (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char            key[1024] = {0};
         char            *ssl_user = NULL;
         char            *value = NULL;
+        char            *address_family_data = NULL;
 
         if (!graph || !volinfo || !set_dict || !brickinfo)
                 goto out;
@@ -2122,6 +2123,17 @@ brick_graph_add_server (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         RPC_SET_OPT(xl, SSL_DH_PARAM_OPT,   "ssl-dh-param",         return -1);
         RPC_SET_OPT(xl, SSL_EC_CURVE_OPT,   "ssl-ec-curve",         return -1);
 
+        if (dict_get_str (volinfo->dict, "transport.address-family",
+                &address_family_data) == 0) {
+                ret = xlator_set_option (xl, "transport.address-family",
+                                address_family_data);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_WARNING,
+                                        "failed to set transport.address-family");
+                        return -1;
+                }
+        }
+
         if (username) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "auth.login.%s.allow",
@@ -2169,6 +2181,7 @@ brick_graph_add_pump (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char            *password      = NULL;
         char            *ptranst       = NULL;
         char            *value         = NULL;
+        char            *address_family_data = NULL;
 
 
         if (!graph || !volinfo || !set_dict)
@@ -2230,6 +2243,18 @@ brick_graph_add_pump (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 if (ret)
                         return -1;
 
+                if (dict_get_str (volinfo->dict, "transport.address-family",
+                                        &address_family_data) == 0) {
+                        ret = xlator_set_option (rbxl,
+                                        "transport.address-family",
+                                        address_family_data);
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_WARNING,
+                                                "failed to set transport.address-family");
+                                return -1;
+                        }
+                }
+
                 xl = volgen_graph_add_nolink (graph, "cluster/pump", "%s-pump",
                                               volinfo->volname);
                 if (!xl)
@@ -2706,6 +2731,7 @@ volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char                    *ssl_str            = NULL;
         gf_boolean_t             ssl_bool           = _gf_false;
         char                    *value              = NULL;
+        char                    *address_family_data = NULL;
 
         GF_ASSERT (graph);
         GF_ASSERT (subvol);
@@ -2735,6 +2761,18 @@ volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 goto err;
 
+        if (dict_get_str (volinfo->dict, "transport.address-family",
+                                &address_family_data) == 0) {
+                ret = xlator_set_option (xl,
+                                "transport.address-family",
+                                address_family_data);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_WARNING,
+                                "failed to set transport.address-family");
+                        goto err;
+                }
+        }
+
         ret = dict_get_uint32 (set_dict, "trusted-client",
                                &client_type);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 8fd24f943ba..1c354493244 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -285,6 +285,7 @@ __glusterd_handle_create_volume (rpcsvc_request_t *req)
         xlator_t               *this        = NULL;
         char                   *free_ptr    = NULL;
         char                   *trans_type  = NULL;
+        char                   *address_family_str  = NULL;
         uuid_t                  volume_id   = {0,};
         uuid_t                  tmp_uuid    = {0};
         int32_t                 type        = 0;
@@ -365,6 +366,8 @@ __glusterd_handle_create_volume (rpcsvc_request_t *req)
                 goto out;
         }
 
+
+
         ret = dict_get_str (dict, "transport", &trans_type);
         if (ret) {
                 snprintf (err_str, sizeof (err_str), "Unable to get "
@@ -373,6 +376,30 @@ __glusterd_handle_create_volume (rpcsvc_request_t *req)
                         GD_MSG_DICT_GET_FAILED, "%s", err_str);
                 goto out;
         }
+
+        ret = dict_get_str (this->options, "transport.address-family",
+                        &address_family_str);
+
+        if (!ret) {
+                ret = dict_set_dynstr_with_alloc (dict,
+                                "transport.address-family",
+                                address_family_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to set transport.address-family");
+                        goto out;
+                }
+        } else if (!strcmp(trans_type, "tcp")) {
+                /* Setting default as inet for trans_type tcp */
+                ret = dict_set_dynstr_with_alloc (dict,
+                                "transport.address-family",
+                                "inet");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to set transport.address-family");
+                        goto out;
+                }
+        }
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
                 snprintf (err_str, sizeof (err_str), "Unable to get bricks for "
@@ -2089,6 +2116,7 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         char                  msg[1024] __attribute__((unused)) = {0, };
         char                 *brick_mount_dir = NULL;
         char                  key[PATH_MAX]   = "";
+        char                 *address_family_str = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -2361,6 +2389,20 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = dict_get_str (dict, "transport.address-family",
+                        &address_family_str);
+
+        if (!ret) {
+                ret = dict_set_dynstr_with_alloc(volinfo->dict,
+                                "transport.address-family", address_family_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set transport.address-family for %s",
+                                        volinfo->volname);
+                        goto out;
+                }
+        }
+
         gd_update_volume_op_versions (volinfo);
 
         volinfo->caps = caps;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index bbfe2d387d2..126913af548 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1624,6 +1624,12 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .option     = "!ssl-ec-curve",
           .op_version = GD_OP_VERSION_3_7_4,
         },
+        { .key         = "transport.address-family",
+          .voltype     = "protocol/server",
+          .option      = "!address-family",
+          .op_version  = GD_OP_VERSION_3_7_4,
+          .type        = NO_DOC,
+        },
 
         /* Performance xlators enable/disbable options */
         { .key         = "performance.write-behind",
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 7f314da69ec..b0af1af7e9d 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -595,7 +595,8 @@ main ()
     fi
 
     [ -r "$volfile_loc" ] || {
-        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        # '%' included to support ipv6 link local addresses
+        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:%.\-]*\):.*/\1/p');
         volume_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
         [ -n "$volume_str" ] && {
             volume_id="$volume_str";
diff --git a/xlators/mount/fuse/utils/mount_glusterfs.in b/xlators/mount/fuse/utils/mount_glusterfs.in
index 3d58c29e0cf..eca84557e87 100755
--- a/xlators/mount/fuse/utils/mount_glusterfs.in
+++ b/xlators/mount/fuse/utils/mount_glusterfs.in
@@ -501,7 +501,8 @@ main ()
     done
 
     [ -r "$volfile_loc" ] || {
-        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        # '%' included to support ipv6 link local addresses
+        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:%.\-]*\):.*/\1/p');
         volume_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
         [ -n "$volume_str" ] && {
             volume_id="$volume_str";