diff options
| -rwxr-xr-x | tests/bugs/bug-1173414-mgmt-v3-remote-lock-failure.t | 34 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-mem-types.h | 3 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-mgmt.c | 162 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-sm.h | 5 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-syncop.c | 170 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-syncop.h | 21 |
6 files changed, 256 insertions, 139 deletions
diff --git a/tests/bugs/bug-1173414-mgmt-v3-remote-lock-failure.t b/tests/bugs/bug-1173414-mgmt-v3-remote-lock-failure.t new file mode 100755 index 00000000000..adc3fe30dd4 --- /dev/null +++ b/tests/bugs/bug-1173414-mgmt-v3-remote-lock-failure.t @@ -0,0 +1,34 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../cluster.rc + +function check_peers { + $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l +} + +cleanup; + +TEST launch_cluster 2; +TEST $CLI_1 peer probe $H2; + +EXPECT_WITHIN $PROBE_TIMEOUT 1 check_peers + +TEST $CLI_1 volume create $V0 $H1:$B1/$V0 +TEST $CLI_1 volume create $V1 $H1:$B1/$V1 +TEST $CLI_1 volume start $V0 +TEST $CLI_1 volume start $V1 + +for i in {1..20} +do + $CLI_1 volume set $V0 diagnostics.client-log-level DEBUG & + $CLI_1 volume set $V1 barrier on + $CLI_2 volume set $V0 diagnostics.client-log-level DEBUG & + $CLI_2 volume set $V1 barrier on +done + +EXPECT_WITHIN $PROBE_TIMEOUT 1 check_peers +TEST $CLI_1 volume status +TEST $CLI_2 volume status + +cleanup; diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h index 95908cf62fc..ed171b69b66 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h +++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h @@ -70,7 +70,8 @@ typedef enum gf_gld_mem_types_ { gf_gld_mt_snap_t = gf_common_mt_end + 54, gf_gld_mt_missed_snapinfo_t = gf_common_mt_end + 55, gf_gld_mt_snap_create_args_t = gf_common_mt_end + 56, - gf_gld_mt_end = gf_common_mt_end + 57, + gf_gld_mt_local_peers_t = gf_common_mt_end + 57, + gf_gld_mt_end = gf_common_mt_end + 58, } gf_gld_mem_types_t; #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c index f26c676cd67..0cdaaaeda9a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c @@ -384,28 +384,25 @@ out: } int -glusterd_mgmt_v3_initiate_lockdown (glusterd_conf_t *conf, glusterd_op_t op, - dict_t *dict, char **op_errstr, int npeers, - gf_boolean_t *is_acquired) +glusterd_mgmt_v3_initiate_lockdown (glusterd_op_t op, dict_t *dict, + char **op_errstr, int npeers, + gf_boolean_t *is_acquired, + struct list_head *peers) { char *volname = NULL; glusterd_peerinfo_t *peerinfo = NULL; int32_t ret = -1; int32_t peer_cnt = 0; struct syncargs args = {0}; - struct list_head *peers = NULL; uuid_t peer_uuid = {0}; xlator_t *this = NULL; this = THIS; GF_ASSERT (this); - GF_ASSERT (conf); GF_ASSERT (dict); GF_ASSERT (op_errstr); GF_ASSERT (is_acquired); - peers = &conf->xaction_peers; - /* Trying to acquire multiple mgmt_v3 locks on local node */ ret = glusterd_multiple_mgmt_v3_lock (dict, MY_UUID); if (ret) { @@ -425,7 +422,7 @@ glusterd_mgmt_v3_initiate_lockdown (glusterd_conf_t *conf, glusterd_op_t op, gd_syncargs_init (&args, NULL); synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { gd_mgmt_v3_lock (op, dict, peerinfo, &args, MY_UUID, peer_uuid); peer_cnt++; @@ -639,26 +636,23 @@ out: } int -glusterd_mgmt_v3_pre_validate (glusterd_conf_t *conf, glusterd_op_t op, - dict_t *req_dict, char **op_errstr, int npeers) +glusterd_mgmt_v3_pre_validate (glusterd_op_t op, dict_t *req_dict, + char **op_errstr, int npeers, + struct list_head *peers) { int32_t ret = -1; int32_t peer_cnt = 0; dict_t *rsp_dict = NULL; glusterd_peerinfo_t *peerinfo = NULL; struct syncargs args = {0}; - struct list_head *peers = NULL; uuid_t peer_uuid = {0}; xlator_t *this = NULL; this = THIS; GF_ASSERT (this); - GF_ASSERT (conf); GF_ASSERT (req_dict); GF_ASSERT (op_errstr); - peers = &conf->xaction_peers; - rsp_dict = dict_new (); if (!rsp_dict) { gf_log (this->name, GF_LOG_ERROR, @@ -710,7 +704,7 @@ glusterd_mgmt_v3_pre_validate (glusterd_conf_t *conf, glusterd_op_t op, gd_syncargs_init (&args, req_dict); synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { gd_mgmt_v3_pre_validate_req (op, req_dict, peerinfo, &args, MY_UUID, peer_uuid); peer_cnt++; @@ -878,26 +872,23 @@ out: } int -glusterd_mgmt_v3_brick_op (glusterd_conf_t *conf, glusterd_op_t op, - dict_t *req_dict, char **op_errstr, int npeers) +glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict, + char **op_errstr, int npeers, + struct list_head *peers) { int32_t ret = -1; int32_t peer_cnt = 0; dict_t *rsp_dict = NULL; glusterd_peerinfo_t *peerinfo = NULL; struct syncargs args = {0}; - struct list_head *peers = NULL; uuid_t peer_uuid = {0}; xlator_t *this = NULL; this = THIS; GF_ASSERT (this); - GF_ASSERT (conf); GF_ASSERT (req_dict); GF_ASSERT (op_errstr); - peers = &conf->xaction_peers; - rsp_dict = dict_new (); if (!rsp_dict) { gf_log (this->name, GF_LOG_ERROR, @@ -940,7 +931,7 @@ glusterd_mgmt_v3_brick_op (glusterd_conf_t *conf, glusterd_op_t op, gd_syncargs_init (&args, NULL); synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { gd_mgmt_v3_brick_op_req (op, req_dict, peerinfo, &args, MY_UUID, peer_uuid); peer_cnt++; @@ -1103,28 +1094,24 @@ out: } int -glusterd_mgmt_v3_commit (glusterd_conf_t *conf, glusterd_op_t op, - dict_t *op_ctx, dict_t *req_dict, - char **op_errstr, int npeers) +glusterd_mgmt_v3_commit (glusterd_op_t op, dict_t *op_ctx, + dict_t *req_dict, char **op_errstr, + int npeers, struct list_head *peers) { int32_t ret = -1; int32_t peer_cnt = 0; dict_t *rsp_dict = NULL; glusterd_peerinfo_t *peerinfo = NULL; struct syncargs args = {0}; - struct list_head *peers = NULL; uuid_t peer_uuid = {0}; xlator_t *this = NULL; this = THIS; GF_ASSERT (this); - GF_ASSERT (conf); GF_ASSERT (op_ctx); GF_ASSERT (req_dict); GF_ASSERT (op_errstr); - peers = &conf->xaction_peers; - rsp_dict = dict_new (); if (!rsp_dict) { gf_log (this->name, GF_LOG_ERROR, @@ -1176,7 +1163,7 @@ glusterd_mgmt_v3_commit (glusterd_conf_t *conf, glusterd_op_t op, gd_syncargs_init (&args, op_ctx); synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { gd_mgmt_v3_commit_req (op, req_dict, peerinfo, &args, MY_UUID, peer_uuid); peer_cnt++; @@ -1307,29 +1294,24 @@ out: } int -glusterd_mgmt_v3_post_validate (glusterd_conf_t *conf, glusterd_op_t op, - int32_t op_ret, dict_t *dict, dict_t *req_dict, - char **op_errstr, int npeers) +glusterd_mgmt_v3_post_validate (glusterd_op_t op, int32_t op_ret, dict_t *dict, + dict_t *req_dict, char **op_errstr, int npeers, + struct list_head *peers) { int32_t ret = -1; int32_t peer_cnt = 0; dict_t *rsp_dict = NULL; glusterd_peerinfo_t *peerinfo = NULL; struct syncargs args = {0}; - struct list_head *peers = NULL; uuid_t peer_uuid = {0}; xlator_t *this = NULL; this = THIS; GF_ASSERT (this); - GF_ASSERT (conf); GF_ASSERT (dict); GF_VALIDATE_OR_GOTO (this->name, req_dict, out); GF_ASSERT (op_errstr); - peers = &conf->xaction_peers; - GF_ASSERT (peers); - rsp_dict = dict_new (); if (!rsp_dict) { gf_log (this->name, GF_LOG_ERROR, @@ -1375,7 +1357,7 @@ glusterd_mgmt_v3_post_validate (glusterd_conf_t *conf, glusterd_op_t op, gd_syncargs_init (&args, req_dict); synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { gd_mgmt_v3_post_validate_req (op, op_ret, req_dict, peerinfo, &args, MY_UUID, peer_uuid); peer_cnt++; @@ -1502,10 +1484,11 @@ out: } int -glusterd_mgmt_v3_release_peer_locks (glusterd_conf_t *conf, glusterd_op_t op, +glusterd_mgmt_v3_release_peer_locks (glusterd_op_t op, dict_t *dict, int32_t op_ret, char **op_errstr, int npeers, - gf_boolean_t is_acquired) + gf_boolean_t is_acquired, + struct list_head *peers) { int32_t ret = -1; int32_t peer_cnt = 0; @@ -1513,16 +1496,12 @@ glusterd_mgmt_v3_release_peer_locks (glusterd_conf_t *conf, glusterd_op_t op, xlator_t *this = NULL; glusterd_peerinfo_t *peerinfo = NULL; struct syncargs args = {0}; - struct list_head *peers = NULL; this = THIS; GF_ASSERT (this); - GF_ASSERT (conf); GF_ASSERT (dict); GF_ASSERT (op_errstr); - peers = &conf->xaction_peers; - /* If the lock has not been held during this * transaction, do not send unlock requests */ if (!is_acquired) @@ -1537,7 +1516,7 @@ glusterd_mgmt_v3_release_peer_locks (glusterd_conf_t *conf, glusterd_op_t op, gd_syncargs_init (&args, NULL); synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { gd_mgmt_v3_unlock (op, dict, peerinfo, &args, MY_UUID, peer_uuid); peer_cnt++; @@ -1575,6 +1554,7 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op, xlator_t *this = NULL; gf_boolean_t is_acquired = _gf_false; uuid_t *originator_uuid = NULL; + struct list_head xaction_peers = {0,}; this = THIS; GF_ASSERT (this); @@ -1583,6 +1563,15 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op, conf = this->private; GF_ASSERT (conf); + INIT_LIST_HEAD (&xaction_peers); + npeers = gd_build_local_xaction_peers_list (&conf->peers, + &xaction_peers, op); + if (npeers == -1) { + gf_log (this->name, GF_LOG_ERROR, "building local peers list " + "failed"); + goto rsp; + } + /* Save the MY_UUID as the originator_uuid. This originator_uuid * will be used by is_origin_glusterd() to determine if a node * is the originator node for a command. */ @@ -1619,13 +1608,10 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op, } dict_copy (dict, tmp_dict); - /* BUILD PEERS LIST */ - INIT_LIST_HEAD (&conf->xaction_peers); - npeers = gd_build_peers_list (&conf->peers, &conf->xaction_peers, op); - /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */ - ret = glusterd_mgmt_v3_initiate_lockdown (conf, op, dict, &op_errstr, - npeers, &is_acquired); + ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr, + npeers, &is_acquired, + &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed."); goto out; @@ -1642,16 +1628,17 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op, } /* PRE-COMMIT VALIDATE PHASE */ - ret = glusterd_mgmt_v3_pre_validate (conf, op, req_dict, - &op_errstr, npeers); + ret = glusterd_mgmt_v3_pre_validate (op, req_dict, + &op_errstr, npeers, + &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed"); goto out; } /* COMMIT OP PHASE */ - ret = glusterd_mgmt_v3_commit (conf, op, dict, req_dict, - &op_errstr, npeers); + ret = glusterd_mgmt_v3_commit (op, dict, req_dict, + &op_errstr, npeers, &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed"); goto out; @@ -1662,8 +1649,9 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op, commands other than snapshot. So as of now, I am sending 0 (op_ret as 0). */ - ret = glusterd_mgmt_v3_post_validate (conf, op, 0, dict, req_dict, - &op_errstr, npeers); + ret = glusterd_mgmt_v3_post_validate (op, 0, dict, req_dict, + &op_errstr, npeers, + &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed"); goto out; @@ -1673,9 +1661,10 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op, out: op_ret = ret; /* UNLOCK PHASE FOR PEERS*/ - (void) glusterd_mgmt_v3_release_peer_locks (conf, op, dict, + (void) glusterd_mgmt_v3_release_peer_locks (op, dict, op_ret, &op_errstr, - npeers, is_acquired); + npeers, is_acquired, + &xaction_peers); /* LOCAL VOLUME(S) UNLOCK */ if (is_acquired) { @@ -1687,10 +1676,12 @@ out: op_ret = ret; } } - +rsp: /* SEND CLI RESPONSE */ glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr); + gd_cleanup_local_xaction_peers_list (&xaction_peers); + if (req_dict) dict_unref (req_dict); @@ -1783,6 +1774,7 @@ glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op, uuid_t *originator_uuid = NULL; gf_boolean_t success = _gf_false; char *cli_errstr = NULL; + struct list_head xaction_peers = {0,}; this = THIS; GF_ASSERT (this); @@ -1791,6 +1783,15 @@ glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op, conf = this->private; GF_ASSERT (conf); + INIT_LIST_HEAD (&xaction_peers); + npeers = gd_build_local_xaction_peers_list (&conf->peers, + &xaction_peers, op); + if (npeers == -1) { + gf_log (this->name, GF_LOG_ERROR, "building local peers list " + "failed"); + goto rsp; + } + /* Save the MY_UUID as the originator_uuid. This originator_uuid * will be used by is_origin_glusterd() to determine if a node * is the originator node for a command. */ @@ -1827,13 +1828,10 @@ glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op, } dict_copy (dict, tmp_dict); - /* BUILD PEERS LIST */ - INIT_LIST_HEAD (&conf->xaction_peers); - npeers = gd_build_peers_list (&conf->peers, &conf->xaction_peers, op); - /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */ - ret = glusterd_mgmt_v3_initiate_lockdown (conf, op, dict, &op_errstr, - npeers, &is_acquired); + ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr, + npeers, &is_acquired, + &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed."); goto out; @@ -1850,8 +1848,8 @@ glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op, } /* PRE-COMMIT VALIDATE PHASE */ - ret = glusterd_mgmt_v3_pre_validate (conf, op, req_dict, - &op_errstr, npeers); + ret = glusterd_mgmt_v3_pre_validate (op, req_dict, + &op_errstr, npeers, &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed"); goto out; @@ -1875,8 +1873,8 @@ glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op, goto out; } - ret = glusterd_mgmt_v3_brick_op (conf, op, req_dict, - &op_errstr, npeers); + ret = glusterd_mgmt_v3_brick_op (op, req_dict, + &op_errstr, npeers, &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed"); goto unbarrier; @@ -1906,8 +1904,8 @@ glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op, goto unbarrier; } - ret = glusterd_mgmt_v3_commit (conf, op, dict, req_dict, - &op_errstr, npeers); + ret = glusterd_mgmt_v3_commit (op, dict, req_dict, + &op_errstr, npeers, &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed"); /* If the main op fails, we should save the error string. @@ -1932,8 +1930,8 @@ unbarrier: goto out; } - ret = glusterd_mgmt_v3_brick_op (conf, op, req_dict, - &op_errstr, npeers); + ret = glusterd_mgmt_v3_brick_op (op, req_dict, + &op_errstr, npeers, &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed"); @@ -1960,17 +1958,19 @@ out: op_ret = -1; /* POST-COMMIT VALIDATE PHASE */ - ret = glusterd_mgmt_v3_post_validate (conf, op, op_ret, dict, req_dict, - &op_errstr, npeers); + ret = glusterd_mgmt_v3_post_validate (op, op_ret, dict, req_dict, + &op_errstr, npeers, + &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed"); op_ret = -1; } /* UNLOCK PHASE FOR PEERS*/ - (void) glusterd_mgmt_v3_release_peer_locks (conf, op, dict, + (void) glusterd_mgmt_v3_release_peer_locks (op, dict, op_ret, &op_errstr, - npeers, is_acquired); + npeers, is_acquired, + &xaction_peers); /* If the commit op (snapshot taking) failed, then the error is stored in cli_errstr and unbarrier is called. Suppose, if unbarrier also @@ -1994,10 +1994,12 @@ out: op_ret = ret; } } - +rsp: /* SEND CLI RESPONSE */ glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr); + gd_cleanup_local_xaction_peers_list (&xaction_peers); + if (req_dict) dict_unref (req_dict); diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h index 294dbacc9d0..fb873f75601 100644 --- a/xlators/mgmt/glusterd/src/glusterd-sm.h +++ b/xlators/mgmt/glusterd/src/glusterd-sm.h @@ -103,6 +103,11 @@ struct glusterd_peerinfo_ { typedef struct glusterd_peerinfo_ glusterd_peerinfo_t; +typedef struct glusterd_local_peers_ { + glusterd_peerinfo_t *peerinfo; + struct list_head op_peers_list; +} glusterd_local_peers_t; + typedef enum glusterd_ev_gen_mode_ { GD_MODE_OFF, GD_MODE_ON, diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c index 534e3032105..a910406f158 100644 --- a/xlators/mgmt/glusterd/src/glusterd-syncop.c +++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c @@ -1030,6 +1030,9 @@ gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers, glusterd_peerinfo_t *peerinfo = NULL; int npeers = 0; + GF_ASSERT (peers); + GF_ASSERT (xact_peers); + list_for_each_entry (peerinfo, peers, uuid_list) { if (!peerinfo->connected) continue; @@ -1044,18 +1047,65 @@ gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers, } int -gd_lock_op_phase (glusterd_conf_t *conf, glusterd_op_t op, dict_t *op_ctx, - char **op_errstr, int npeers, uuid_t txn_id) +gd_build_local_xaction_peers_list (struct list_head *peers, + struct list_head *xact_peers, + glusterd_op_t op) { - int ret = -1; - int peer_cnt = 0; - uuid_t peer_uuid = {0}; - xlator_t *this = NULL; - glusterd_peerinfo_t *peerinfo = NULL; - struct syncargs args = {0}; - struct list_head *peers = NULL; + glusterd_peerinfo_t *peerinfo = NULL; + glusterd_local_peers_t *local_peers = NULL; + int npeers = 0; + + GF_ASSERT (peers); + GF_ASSERT (xact_peers); - peers = &conf->xaction_peers; + list_for_each_entry (peerinfo, peers, uuid_list) { + if (!peerinfo->connected) + continue; + if (op != GD_OP_SYNC_VOLUME && + peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) + continue; + + local_peers = GF_CALLOC (1, sizeof (*local_peers), + gf_gld_mt_local_peers_t); + if (!local_peers) { + return -1; + } + INIT_LIST_HEAD (&local_peers->op_peers_list); + local_peers->peerinfo = peerinfo; + list_add_tail (&local_peers->op_peers_list, xact_peers); + npeers++; + } + return npeers; +} + +void +gd_cleanup_local_xaction_peers_list (struct list_head *xact_peers) +{ + GF_ASSERT (xact_peers); + + if (list_empty (xact_peers)) + return; + + glusterd_local_peers_t *local_peers = NULL; + + list_for_each_entry (local_peers, xact_peers, op_peers_list) { + GF_FREE (local_peers); + /* local_peers->peerinfo need not be freed because it does not + * ownership of peerinfo, but merely refer it */ + } +} + +int +gd_lock_op_phase (glusterd_conf_t *conf, glusterd_op_t op, dict_t *op_ctx, + char **op_errstr, int npeers, uuid_t txn_id, + struct list_head *peers) +{ + int ret = -1; + int peer_cnt = 0; + uuid_t peer_uuid = {0}; + xlator_t *this = NULL; + glusterd_peerinfo_t *peerinfo = NULL; + struct syncargs args = {0}; if (!npeers) { ret = 0; @@ -1065,7 +1115,7 @@ gd_lock_op_phase (glusterd_conf_t *conf, glusterd_op_t op, dict_t *op_ctx, this = THIS; synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { if (conf->op_version < GD_OP_VERSION_3_6_0) { /* Reset lock status */ peerinfo->locked = _gf_false; @@ -1106,16 +1156,16 @@ int gd_stage_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, char **op_errstr, int npeers) { - int ret = -1; - int peer_cnt = 0; - dict_t *rsp_dict = NULL; - char *hostname = NULL; - xlator_t *this = NULL; - glusterd_peerinfo_t *peerinfo = NULL; - uuid_t tmp_uuid = {0}; - char *errstr = NULL; - struct syncargs args = {0}; - dict_t *aggr_dict = NULL; + int ret = -1; + int peer_cnt = 0; + dict_t *rsp_dict = NULL; + char *hostname = NULL; + xlator_t *this = NULL; + glusterd_peerinfo_t *peerinfo = NULL; + uuid_t tmp_uuid = {0}; + char *errstr = NULL; + struct syncargs args = {0}; + dict_t *aggr_dict = NULL; this = THIS; rsp_dict = dict_new (); @@ -1165,7 +1215,8 @@ stage_done: gd_syncargs_init (&args, aggr_dict); synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + + list_for_each_local_xaction_peers (peerinfo, peers) { ret = gd_syncop_mgmt_stage_op (peerinfo, &args, MY_UUID, tmp_uuid, op, req_dict, op_ctx); @@ -1202,16 +1253,16 @@ int gd_commit_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, char **op_errstr, int npeers) { - dict_t *rsp_dict = NULL; - int peer_cnt = -1; - int ret = -1; - char *hostname = NULL; - glusterd_peerinfo_t *peerinfo = NULL; - xlator_t *this = NULL; - uuid_t tmp_uuid = {0}; - char *errstr = NULL; - struct syncargs args = {0}; - int type = GF_QUOTA_OPTION_TYPE_NONE; + dict_t *rsp_dict = NULL; + int peer_cnt = -1; + int ret = -1; + char *hostname = NULL; + glusterd_peerinfo_t *peerinfo = NULL; + xlator_t *this = NULL; + uuid_t tmp_uuid = {0}; + char *errstr = NULL; + struct syncargs args = {0}; + int type = GF_QUOTA_OPTION_TYPE_NONE; this = THIS; rsp_dict = dict_new (); @@ -1269,7 +1320,8 @@ commit_done: gd_syncargs_init (&args, op_ctx); synctask_barrier_init((&args)); peer_cnt = 0; - list_for_each_entry (peerinfo, peers, op_peers_list) { + + list_for_each_local_xaction_peers (peerinfo, peers) { ret = gd_syncop_mgmt_commit_op (peerinfo, &args, MY_UUID, tmp_uuid, op, req_dict, op_ctx); @@ -1301,18 +1353,15 @@ int gd_unlock_op_phase (glusterd_conf_t *conf, glusterd_op_t op, int *op_ret, rpcsvc_request_t *req, dict_t *op_ctx, char *op_errstr, int npeers, char *volname, gf_boolean_t is_acquired, - uuid_t txn_id) + uuid_t txn_id, struct list_head *peers) { - glusterd_peerinfo_t *peerinfo = NULL; - glusterd_peerinfo_t *tmp = NULL; - uuid_t tmp_uuid = {0}; - int peer_cnt = 0; - int ret = -1; - xlator_t *this = NULL; - struct syncargs args = {0}; - struct list_head *peers = NULL; - - peers = &conf->xaction_peers; + glusterd_peerinfo_t *peerinfo = NULL; + uuid_t tmp_uuid = {0}; + int peer_cnt = 0; + int ret = -1; + xlator_t *this = NULL; + struct syncargs args = {0}; + this = THIS; GF_ASSERT (this); @@ -1330,25 +1379,23 @@ gd_unlock_op_phase (glusterd_conf_t *conf, glusterd_op_t op, int *op_ret, synctask_barrier_init((&args)); peer_cnt = 0; + if (conf->op_version < GD_OP_VERSION_3_6_0) { - list_for_each_entry_safe (peerinfo, tmp, peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { /* Only unlock peers that were locked */ if (peerinfo->locked) { gd_syncop_mgmt_unlock (peerinfo, &args, MY_UUID, tmp_uuid); peer_cnt++; - list_del_init (&peerinfo->op_peers_list); } } } else { if (volname) { - list_for_each_entry_safe (peerinfo, tmp, - peers, op_peers_list) { + list_for_each_local_xaction_peers (peerinfo, peers) { gd_syncop_mgmt_v3_unlock (op_ctx, peerinfo, &args, MY_UUID, tmp_uuid, txn_id); peer_cnt++; - list_del_init (&peerinfo->op_peers_list); } } } @@ -1502,6 +1549,7 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req) xlator_t *this = NULL; gf_boolean_t is_acquired = _gf_false; uuid_t *txn_id = NULL; + struct list_head xaction_peers = {0,}; glusterd_op_info_t txn_opinfo; this = THIS; @@ -1509,6 +1557,8 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req) conf = this->private; GF_ASSERT (conf); + INIT_LIST_HEAD (&xaction_peers); + ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Failed to get volume " @@ -1591,15 +1641,20 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req) local_locking_done: - INIT_LIST_HEAD (&conf->xaction_peers); - - npeers = gd_build_peers_list (&conf->peers, &conf->xaction_peers, op); + /* Maintain xaction_peers on per transaction basis */ + npeers = gd_build_local_xaction_peers_list (&conf->peers, + &xaction_peers, op); + if (npeers == -1) { + gf_log (this->name, GF_LOG_ERROR, "building local peers list " + "failed"); + goto out; + } /* If no volname is given as a part of the command, locks will * not be held */ if (volname || (conf->op_version < GD_OP_VERSION_3_6_0)) { ret = gd_lock_op_phase (conf, op, op_ctx, &op_errstr, - npeers, *txn_id); + npeers, *txn_id, &xaction_peers); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Locking Peers Failed."); @@ -1616,7 +1671,7 @@ local_locking_done: goto out; } - ret = gd_stage_op_phase (&conf->xaction_peers, op, op_ctx, req_dict, + ret = gd_stage_op_phase (&xaction_peers, op, op_ctx, req_dict, &op_errstr, npeers); if (ret) goto out; @@ -1625,7 +1680,7 @@ local_locking_done: if (ret) goto out; - ret = gd_commit_op_phase (&conf->xaction_peers, op, op_ctx, req_dict, + ret = gd_commit_op_phase (&xaction_peers, op, op_ctx, req_dict, &op_errstr, npeers); if (ret) goto out; @@ -1637,7 +1692,8 @@ out: (void) gd_unlock_op_phase (conf, op, &op_ret, req, op_ctx, op_errstr, npeers, volname, - is_acquired, *txn_id); + is_acquired, *txn_id, + &xaction_peers); /* Clearing the transaction opinfo */ ret = glusterd_clear_txn_opinfo (txn_id); @@ -1650,6 +1706,8 @@ out: glusterd_op_send_cli_response (op, op_ret, 0, req, op_ctx, op_errstr); + gd_cleanup_local_xaction_peers_list (&xaction_peers); + if (volname) GF_FREE (volname); diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h index c7924b5a025..e0733dc8006 100644 --- a/xlators/mgmt/glusterd/src/glusterd-syncop.h +++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h @@ -37,11 +37,19 @@ } while (0) +#define list_for_each_local_xaction_peers(xact_peer, xact_peers_head) \ + glusterd_local_peers_t *pos = NULL; \ + for (pos = list_entry((xact_peers_head)->next, glusterd_local_peers_t,\ + op_peers_list), \ + xact_peer = pos->peerinfo; \ + &pos->op_peers_list != (xact_peers_head); \ + pos = list_entry(pos->op_peers_list.next, glusterd_local_peers_t,\ + op_peers_list), \ + xact_peer = pos->peerinfo) + int gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local, void *cookie, rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc); - - int gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg, uuid_t my_uuid, uuid_t recv_uuid); @@ -64,6 +72,15 @@ gd_synctask_barrier_wait (struct syncargs *args, int count); int gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers, glusterd_op_t op); + +int +gd_build_local_xaction_peers_list (struct list_head *peers, + struct list_head *xact_peers, + glusterd_op_t op); + +void +gd_cleanup_local_xaction_peers_list (struct list_head *peers); + int gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, char **op_errstr); |