From e3d97f57c8e25e8b44d3c96b09d69336ff6edb4b Mon Sep 17 00:00:00 2001 From: Mohit Agrawal Date: Wed, 6 Nov 2019 10:32:04 +0530 Subject: glusterd: Client Handling of Elastic Clusters Configure the list of gluster servers in the key GLUSTERD_BRICK_SERVERS at the time of GETSPEC RPC CALL and access the value in client side to update volfile serve list so that client would be able to connect next volfile server in case of current volfile server is down Updates #741 Signed-off-by: Mohit Agrawal Change-Id: I23f36ddb92982bb02ffd83937a8bd8a2c97e8104 --- glusterfsd/src/glusterfsd-mgmt.c | 6 ++- libglusterfs/src/glusterfs/glusterfs.h | 2 +- tests/bugs/glusterd/check_elastic_server.t | 60 ++++++++++++++++++++++++++ tests/cluster.rc | 5 +++ xlators/mgmt/glusterd/src/glusterd-handshake.c | 49 +++++++++++++++++++++ 5 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 tests/bugs/glusterd/check_elastic_server.t diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c index bacb0ae2824..43cc837f029 100644 --- a/glusterfsd/src/glusterfsd-mgmt.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -2153,10 +2153,12 @@ mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count, } dict->extra_stdfree = rsp.xdata.xdata_val; - /* glusterd2 only */ ret = dict_get_str(dict, "servers-list", &servers_list); if (ret) { - goto volfile; + /* Server list is set by glusterd at the time of getspec */ + ret = dict_get_str(dict, GLUSTERD_BRICK_SERVERS, &servers_list); + if (ret) + goto volfile; } gf_log(frame->this->name, GF_LOG_INFO, diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h index 0b2d7b18efe..ea024a02a30 100644 --- a/libglusterfs/src/glusterfs/glusterfs.h +++ b/libglusterfs/src/glusterfs/glusterfs.h @@ -77,7 +77,7 @@ #define GLUSTERD_MAX_SNAP_NAME 255 #define GLUSTERFS_SOCKET_LISTEN_BACKLOG 1024 - +#define GLUSTERD_BRICK_SERVERS "cluster.brick-vol-servers" #define SLEN(str) (sizeof(str) - 1) #define ZR_MOUNTPOINT_OPT "mountpoint" diff --git a/tests/bugs/glusterd/check_elastic_server.t b/tests/bugs/glusterd/check_elastic_server.t new file mode 100644 index 00000000000..8e9e4e5b0eb --- /dev/null +++ b/tests/bugs/glusterd/check_elastic_server.t @@ -0,0 +1,60 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../cluster.rc +. $(dirname $0)/../../volume.rc + +function cluster_rebalance_status { + local vol=$1 + $CLI_2 volume status | grep -iw "Rebalance" -A 5 | grep "Status" | sed 's/.*: //' +} + +cleanup; +TEST launch_cluster 4; +TEST $CLI_1 peer probe $H2; +TEST $CLI_1 peer probe $H3; +TEST $CLI_1 peer probe $H4; + +EXPECT_WITHIN $PROBE_TIMEOUT 3 peer_count + +TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0 +EXPECT 'Created' cluster_volinfo_field 1 $V0 'Status'; + +$CLI_1 volume start $V0 +EXPECT 'Started' cluster_volinfo_field 1 $V0 'Status'; + +#Mount FUSE +TEST glusterfs -s $H1 --volfile-id=$V0 $M0; + +TEST mkdir $M0/dir{1..4}; +TEST touch $M0/dir{1..4}/files{1..4}; + +TEST $CLI_1 volume remove-brick $V0 $H1:$B1/$V0 start +EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" cluster_remove_brick_status_completed_field "$V0 $H1:$B1/$V0" + +TEST $CLI_1 volume remove-brick $V0 $H1:$B1/$V0 commit + +kill_glusterd 1 + +total_files=`find $M0 -name "files*" | wc -l` +TEST [ $total_files -eq 16 ]; + +TEST $CLI_2 volume add-brick $V0 $H3:$B3/$V0 + +TEST $CLI_2 volume rebalance $V0 start +EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" cluster_rebalance_status $V0 + +total_files=`find $M0 -name "files*" | wc -l` +TEST [ $total_files -eq 16 ]; + +TEST $CLI_2 volume add-brick $V0 $H4:$B4/$V0 + +TEST $CLI_2 volume rebalance $V0 start +EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" cluster_rebalance_status $V0 +kill_glusterd 2 + +total_files=`find $M0 -name "files*" | wc -l` +TEST [ $total_files -eq 16 ]; + +cleanup; + diff --git a/tests/cluster.rc b/tests/cluster.rc index 06f329e0ef6..2d826d07e39 100644 --- a/tests/cluster.rc +++ b/tests/cluster.rc @@ -192,3 +192,8 @@ function cluster_brick_up_status { eval \$CLI_$1 volume status $vol $host:$brick --xml | sed -ne 's/.*\([01]\)<\/status>/\1/p' } +function cluster_remove_brick_status_completed_field { + local vol=$1 + local brick_list=$2 + $CLI_1 volume remove-brick $vol $brick_list status | awk '{print $7}' | sed -n 3p +} diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c index 1671c2481dc..1cbdc304916 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handshake.c +++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c @@ -907,10 +907,20 @@ __server_getspec(rpcsvc_request_t *req) peer_info_t *peerinfo = NULL; xlator_t *this = NULL; dict_t *dict = NULL; + glusterd_peerinfo_t *peer = NULL; + glusterd_conf_t *conf = NULL; + int peer_cnt = 0; + char *peer_hosts = NULL; + char *tmp_str = NULL; + char portstr[10] = { + 0, + }; + int len = 0; this = THIS; GF_ASSERT(this); + conf = this->private; ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_getspec_req); if (ret < 0) { // failed to decode msg; @@ -998,6 +1008,43 @@ __server_getspec(rpcsvc_request_t *req) dict); } + RCU_READ_LOCK; + cds_list_for_each_entry_rcu(peer, &conf->peers, uuid_list) + { + if (!peer->connected) + continue; + if (!peer_hosts) { + if (peer->port) { + snprintf(portstr, sizeof(portstr), "%d", peer->port); + } else { + snprintf(portstr, sizeof(portstr), "%d", GLUSTERD_DEFAULT_PORT); + } + len = strlen(peer->hostname) + strlen(portstr) + 3; + tmp_str = GF_CALLOC(1, len, gf_gld_mt_char); + snprintf(tmp_str, len, "%s%s%s%s", peer->hostname, ":", portstr, + " "); + peer_hosts = tmp_str; + } else { + len = strlen(peer_hosts) + strlen(peer->hostname) + + strlen(portstr) + 3; + tmp_str = GF_CALLOC(1, len, gf_gld_mt_char); + snprintf(tmp_str, len, "%s%s%s%s%s", peer_hosts, peer->hostname, + ":", portstr, " "); + GF_FREE(peer_hosts); + peer_hosts = tmp_str; + } + peer_cnt++; + } + RCU_READ_UNLOCK; + if (peer_cnt) { + ret = dict_set_str(dict, GLUSTERD_BRICK_SERVERS, peer_hosts); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "failed to set peer_host in dict"); + goto fail; + } + } + if (ret == 0) { if (dict->count > 0) { ret = dict_allocate_and_serialize(dict, &rsp.xdata.xdata_val, @@ -1076,6 +1123,8 @@ fail: free(args.key); // malloced by xdr free(rsp.spec); + if (peer_hosts) + GF_FREE(peer_hosts); if (dict) dict_unref(dict); -- cgit