From cd5c9df4b67073089a5502d5e4c2e2e7d2141e4a Mon Sep 17 00:00:00 2001 From: Pavan Sondur Date: Fri, 8 Oct 2010 07:06:57 +0000 Subject: mgmt/glusterd: Fix replace brick to pass src-brick and dst-brick port nos properly. Fix also contains patch from shishir ng for maintaining replace-brick states in glusterd. Signed-off-by: Pavan Vilas Sondur Signed-off-by: Vijay Bellur BUG: 1235 (Bug for all pump/migrate commits) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1235 --- rpc/xdr/src/glusterd1-xdr.c | 6 + rpc/xdr/src/glusterd1-xdr.h | 4 + rpc/xdr/src/glusterd1.x | 1 + xlators/cluster/afr/src/pump.c | 17 +- xlators/mgmt/glusterd/src/glusterd-handler.c | 16 +- xlators/mgmt/glusterd/src/glusterd-op-sm.c | 432 +++++++++++++++++-------- xlators/mgmt/glusterd/src/glusterd-op-sm.h | 3 +- xlators/mgmt/glusterd/src/glusterd-rebalance.c | 7 + xlators/mgmt/glusterd/src/glusterd-utils.c | 50 +++ xlators/mgmt/glusterd/src/glusterd-utils.h | 12 + xlators/mgmt/glusterd/src/glusterd.h | 15 +- xlators/mgmt/glusterd/src/glusterd3_1-mops.c | 133 +++++--- 12 files changed, 510 insertions(+), 186 deletions(-) diff --git a/rpc/xdr/src/glusterd1-xdr.c b/rpc/xdr/src/glusterd1-xdr.c index fce8b87cb63..189c1a56ee2 100644 --- a/rpc/xdr/src/glusterd1-xdr.c +++ b/rpc/xdr/src/glusterd1-xdr.c @@ -219,6 +219,8 @@ xdr_gd1_mgmt_stage_op_rsp (XDR *xdrs, gd1_mgmt_stage_op_rsp *objp) } if (!xdr_string (xdrs, &objp->op_errstr, ~0)) return FALSE; + if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0)) + return FALSE; return TRUE; } else if (xdrs->x_op == XDR_DECODE) { if (!xdr_vector (xdrs, (char *)objp->uuid, 16, @@ -240,6 +242,8 @@ xdr_gd1_mgmt_stage_op_rsp (XDR *xdrs, gd1_mgmt_stage_op_rsp *objp) } if (!xdr_string (xdrs, &objp->op_errstr, ~0)) return FALSE; + if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0)) + return FALSE; return TRUE; } @@ -254,6 +258,8 @@ xdr_gd1_mgmt_stage_op_rsp (XDR *xdrs, gd1_mgmt_stage_op_rsp *objp) return FALSE; if (!xdr_string (xdrs, &objp->op_errstr, ~0)) return FALSE; + if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0)) + return FALSE; return TRUE; } diff --git a/rpc/xdr/src/glusterd1-xdr.h b/rpc/xdr/src/glusterd1-xdr.h index 74e4d9d7656..ca57059d829 100644 --- a/rpc/xdr/src/glusterd1-xdr.h +++ b/rpc/xdr/src/glusterd1-xdr.h @@ -133,6 +133,10 @@ struct gd1_mgmt_stage_op_rsp { int op_ret; int op_errno; char *op_errstr; + struct { + u_int dict_len; + char *dict_val; + } dict; }; typedef struct gd1_mgmt_stage_op_rsp gd1_mgmt_stage_op_rsp; diff --git a/rpc/xdr/src/glusterd1.x b/rpc/xdr/src/glusterd1.x index ad6d32219e2..ff436729c28 100644 --- a/rpc/xdr/src/glusterd1.x +++ b/rpc/xdr/src/glusterd1.x @@ -80,6 +80,7 @@ struct gd1_mgmt_stage_op_rsp { int op_ret; int op_errno; string op_errstr<>; + opaque dict<>; } ; struct gd1_mgmt_commit_op_req { diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c index 9a3a25af39a..a499cbb330d 100644 --- a/xlators/cluster/afr/src/pump.c +++ b/xlators/cluster/afr/src/pump.c @@ -1096,14 +1096,23 @@ out: int pump_execute_abort (call_frame_t *frame, xlator_t *this) { - afr_private_t *priv = NULL; - afr_local_t *local = NULL; + afr_private_t *priv = NULL; + pump_private_t *pump_priv = NULL; + afr_local_t *local = NULL; - priv = this->private; - local = frame->local; + priv = this->private; + pump_priv = priv->pump_private; + local = frame->local; pump_change_state (this, PUMP_STATE_ABORT); + LOCK (&pump_priv->resume_path_lock); + { + pump_priv->number_files_pumped = 0; + pump_priv->current_file[0] = '\0'; + } + UNLOCK (&pump_priv->resume_path_lock); + local->op_ret = 0; pump_command_reply (frame, this); diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index 7e5cddac88d..70734074fa7 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -1890,11 +1890,12 @@ out: int glusterd_op_stage_send_resp (rpcsvc_request_t *req, - int32_t op, int32_t status, char *op_errstr) + int32_t op, int32_t status, + char *op_errstr, dict_t *rsp_dict) { - gd1_mgmt_stage_op_rsp rsp = {{0},}; - int ret = -1; + gd1_mgmt_stage_op_rsp rsp = {{0},}; + int ret = -1; GF_ASSERT (req); rsp.op_ret = status; @@ -1905,6 +1906,15 @@ glusterd_op_stage_send_resp (rpcsvc_request_t *req, else rsp.op_errstr = ""; + ret = dict_allocate_and_serialize (rsp_dict, + &rsp.dict.dict_val, + (size_t *)&rsp.dict.dict_len); + if (ret < 0) { + gf_log ("", GF_LOG_DEBUG, + "failed to get serialized length of dict"); + return ret; + } + ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL, gd_xdr_serialize_mgmt_stage_op_rsp); diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index d9d324b2e1e..34caacd5ea8 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -828,7 +828,8 @@ out: } static int -glusterd_op_stage_replace_brick (gd1_mgmt_stage_op_req *req, char **op_errstr) +glusterd_op_stage_replace_brick (gd1_mgmt_stage_op_req *req, char **op_errstr, + dict_t *rsp_dict) { int ret = 0; dict_t *dict = NULL; @@ -844,6 +845,7 @@ glusterd_op_stage_replace_brick (gd1_mgmt_stage_op_req *req, char **op_errstr) char *dup_dstbrick = NULL; glusterd_peerinfo_t *peerinfo = NULL; struct stat st_buf = {0,}; + glusterd_brickinfo_t *dst_brickinfo = NULL; GF_ASSERT (req); @@ -917,6 +919,56 @@ glusterd_op_stage_replace_brick (gd1_mgmt_stage_op_req *req, char **op_errstr) goto out; } + switch (replace_op) { + case GF_REPLACE_OP_START: + if (glusterd_is_rb_started (volinfo)) { + gf_log ("", GF_LOG_ERROR, "Replace brick is already " + "started for volume "); + ret = -1; + goto out; + } + break; + case GF_REPLACE_OP_PAUSE: + if (glusterd_is_rb_paused (volinfo)) { + gf_log ("", GF_LOG_ERROR, "Replace brick is already" + " paused for volume "); + ret = -1; + goto out; + } else if (!glusterd_is_rb_started(volinfo)) { + gf_log ("", GF_LOG_ERROR, "Replace brick is not" + " started for volume "); + ret = -1; + goto out; + } + break; + + case GF_REPLACE_OP_ABORT: + if ((!glusterd_is_rb_paused (volinfo)) && + (!glusterd_is_rb_started (volinfo))) { + gf_log ("", GF_LOG_ERROR, "Replace brick is not " + " started or paused for volume "); + ret = -1; + goto out; + } + break; + + case GF_REPLACE_OP_COMMIT: + if (!glusterd_is_rb_started (volinfo)) { + gf_log ("", GF_LOG_ERROR, "Replace brick is not " + "started for volume "); + ret = -1; + goto out; + } + break; + + case GF_REPLACE_OP_COMMIT_FORCE: break; + case GF_REPLACE_OP_STATUS: + break; + default: + ret = -1; + goto out; + } + ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo, &src_brickinfo); if (ret) { @@ -929,6 +981,16 @@ glusterd_op_stage_replace_brick (gd1_mgmt_stage_op_req *req, char **op_errstr) if (!glusterd_is_local_addr (src_brickinfo->hostname)) { gf_log ("", GF_LOG_DEBUG, "I AM THE SOURCE HOST"); + if (src_brickinfo->port) { + ret = dict_set_int32 (rsp_dict, "src-brick-port", + src_brickinfo->port); + if (ret) { + gf_log ("", GF_LOG_DEBUG, + "Could not set src-brick-port=%d", + src_brickinfo->port); + } + } + } dup_dstbrick = gf_strdup (dst_brick); @@ -955,6 +1017,19 @@ glusterd_op_stage_replace_brick (gd1_mgmt_stage_op_req *req, char **op_errstr) goto out; } + ret = glusterd_brickinfo_from_brick (dst_brick, &dst_brickinfo); + if ((volinfo->rb_status ==GF_RB_STATUS_NONE) && + (replace_op == GF_REPLACE_OP_START)) { + volinfo->src_brick = src_brickinfo; + volinfo->dst_brick = dst_brickinfo; + } + + if (glusterd_rb_check_bricks (volinfo, src_brickinfo, dst_brickinfo)) { + gf_log ("", GF_LOG_ERROR, "replace brick: incorrect source or" + " destination bricks specified"); + ret = -1; + goto out; + } if (!glusterd_is_local_addr (host)) { ret = stat (path, &st_buf); if (ret == -1) { @@ -2669,6 +2744,114 @@ out: return ret; } +/* Set src-brick's port number to be used in the maintainance mount + * after all commit acks are received. + */ +static int +rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict, + dict_t *req_dict, int32_t replace_op) +{ + xlator_t *this = NULL; + dict_t *ctx = NULL; + int ret = 0; + int dict_ret = 0; + int src_port = 0; + + this = THIS; + + ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); + if (ctx) { + dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port); + if (src_port) + src_brickinfo->port = src_port; + } + + if (!glusterd_is_local_addr (src_brickinfo->hostname)) { + gf_log ("", GF_LOG_NORMAL, + "adding src-brick port no"); + + src_brickinfo->port = pmap_registry_search (this, + src_brickinfo->path, GF_PMAP_PORT_BRICKSERVER); + if (!src_brickinfo->port && + replace_op != GF_REPLACE_OP_COMMIT_FORCE ) { + gf_log ("", GF_LOG_ERROR, + "Src brick port not available"); + ret = -1; + goto out; + } + + if (rsp_dict) { + ret = dict_set_int32 (rsp_dict, "src-brick-port", src_brickinfo->port); + if (ret) { + gf_log ("", GF_LOG_DEBUG, + "Could not set src-brick port no"); + goto out; + } + } + + ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); + if (ctx) { + ret = dict_set_int32 (ctx, "src-brick-port", src_brickinfo->port); + if (ret) { + gf_log ("", GF_LOG_DEBUG, + "Could not set src-brick port no"); + goto out; + } + } + + } + +out: + return ret; + +} + +static int +rb_update_dstbrick_port (glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict, + dict_t *req_dict, int32_t replace_op) +{ + dict_t *ctx = NULL; + int ret = 0; + int dict_ret = 0; + int dst_port = 0; + + ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); + if (ctx) { + dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port); + if (dst_port) + dst_brickinfo->port = dst_port; + + } + + if (!glusterd_is_local_addr (dst_brickinfo->hostname)) { + gf_log ("", GF_LOG_NORMAL, + "adding dst-brick port no"); + + if (rsp_dict) { + ret = dict_set_int32 (rsp_dict, "dst-brick-port", + dst_brickinfo->port); + if (ret) { + gf_log ("", GF_LOG_DEBUG, + "Could not set dst-brick port no in rsp dict"); + goto out; + } + } + + ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); + if (ctx) { + ret = dict_set_int32 (ctx, "dst-brick-port", + dst_brickinfo->port); + if (ret) { + gf_log ("", GF_LOG_DEBUG, + "Could not set dst-brick port no"); + goto out; + } + } + } +out: + return ret; +} + static int glusterd_op_replace_brick (gd1_mgmt_stage_op_req *req, dict_t *rsp_dict) { @@ -2758,43 +2941,18 @@ glusterd_op_replace_brick (gd1_mgmt_stage_op_req *req, dict_t *rsp_dict) gf_log ("", GF_LOG_DEBUG, "Unable to resolve dst-brickinfo"); goto out; } - /* Set src-brick's port number to be used in the maintainance mount - * after all commit acks are received. - */ - if (!glusterd_is_local_addr (src_brickinfo->hostname)) { - gf_log ("", GF_LOG_NORMAL, - "adding src-brick port no"); - src_brickinfo->port = pmap_registry_search (this, - src_brickinfo->path, GF_PMAP_PORT_BRICKSERVER); - if (!src_brickinfo->port && - replace_op != GF_REPLACE_OP_COMMIT_FORCE ) { - gf_log ("", GF_LOG_ERROR, - "Src brick port not available"); - ret = -1; - goto out; - } - - if (rsp_dict) { - ret = dict_set_int32 (rsp_dict, "src-brick-port", src_brickinfo->port); - if (ret) { - gf_log ("", GF_LOG_DEBUG, - "Could not set src-brick port no"); - goto out; - } - } else { - ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); - GF_ASSERT (ctx); - - ret = dict_set_int32 (ctx, "src-brick-port", src_brickinfo->port); - if (ret) { - gf_log ("", GF_LOG_DEBUG, - "Could not set src-brick port no"); - goto out; - } - } + ret = rb_update_srcbrick_port (src_brickinfo, rsp_dict, + dict, replace_op); + if (ret) + goto out; - } + if ((GF_REPLACE_OP_START != replace_op)) { + ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict, + dict, replace_op); + if (ret) + goto out; + } switch (replace_op) { case GF_REPLACE_OP_START: @@ -2802,40 +2960,19 @@ glusterd_op_replace_brick (gd1_mgmt_stage_op_req *req, dict_t *rsp_dict) if (!glusterd_is_local_addr (dst_brickinfo->hostname)) { gf_log ("", GF_LOG_NORMAL, "I AM THE DESTINATION HOST"); - ret = rb_spawn_destination_brick (volinfo, dst_brickinfo); - if (ret) { - gf_log ("", GF_LOG_DEBUG, - "Failed to spawn destination brick"); - goto out; - } - - if (!glusterd_is_local_addr (dst_brickinfo->hostname)) { - gf_log ("", GF_LOG_NORMAL, - "adding dst-brick port no"); - - if (rsp_dict) { - ret = dict_set_int32 (rsp_dict, "dst-brick-port", - dst_brickinfo->port); - if (ret) { - gf_log ("", GF_LOG_DEBUG, - "Could not set dst-brick port no"); - goto out; - } - } else { - ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); - GF_ASSERT (ctx); - - ret = dict_set_int32 (ctx, "dst-brick-port", - dst_brickinfo->port); - if (ret) { - gf_log ("", GF_LOG_DEBUG, - "Could not set dst-brick port no"); - goto out; - } + if (!glusterd_is_rb_paused (volinfo)) { + ret = rb_spawn_destination_brick (volinfo, dst_brickinfo); + if (ret) { + gf_log ("", GF_LOG_DEBUG, + "Failed to spawn destination brick"); + goto out; } - + } else { + gf_log ("", GF_LOG_ERROR, "Replace brick is already " + "started=> no need to restart dst brick "); } - } + } + if (!glusterd_is_local_addr (src_brickinfo->hostname)) { ret = rb_src_brick_restart (volinfo, src_brickinfo, @@ -2846,13 +2983,24 @@ glusterd_op_replace_brick (gd1_mgmt_stage_op_req *req, dict_t *rsp_dict) goto out; } } + + if (!glusterd_is_local_addr (dst_brickinfo->hostname)) { + gf_log ("", GF_LOG_NORMAL, + "adding dst-brick port no"); + + ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict, + dict, replace_op); + if (ret) + goto out; + } + + glusterd_set_rb_status (volinfo, GF_RB_STATUS_STARTED); break; } case GF_REPLACE_OP_COMMIT: case GF_REPLACE_OP_COMMIT_FORCE: { - ret = dict_set_int32 (volinfo->dict, "enable-pump", 0); gf_log ("", GF_LOG_DEBUG, "Received commit - will be adding dst brick and " @@ -2876,65 +3024,67 @@ glusterd_op_replace_brick (gd1_mgmt_stage_op_req *req, dict_t *rsp_dict) goto out; } - ret = glusterd_op_perform_replace_brick (volinfo, src_brick, - dst_brick); - if (ret) { - gf_log ("", GF_LOG_CRITICAL, "Unable to add " - "dst-brick: %s to volume: %s", - dst_brick, volinfo->volname); - goto out; - } - volinfo->version++; - volinfo->defrag_status = 0; + ret = glusterd_op_perform_replace_brick (volinfo, src_brick, + dst_brick); + if (ret) { + gf_log ("", GF_LOG_CRITICAL, "Unable to add " + "dst-brick: %s to volume: %s", + dst_brick, volinfo->volname); + goto out; + } - ret = glusterd_store_update_volume (volinfo); + volinfo->version++; + volinfo->defrag_status = 0; - if (ret) - goto out; + ret = glusterd_store_update_volume (volinfo); - ret = glusterd_volume_compute_cksum (volinfo); - if (ret) - goto out; + if (ret) + goto out; - ret = glusterd_check_generate_start_nfs (volinfo); + ret = glusterd_volume_compute_cksum (volinfo); + if (ret) + goto out; - if (ret) { - gf_log ("", GF_LOG_CRITICAL, "Failed to generate " - " nfs volume file"); - } + ret = glusterd_check_generate_start_nfs (volinfo); - ret = glusterd_fetchspec_notify (THIS); + if (ret) { + gf_log ("", GF_LOG_CRITICAL, "Failed to generate " + " nfs volume file"); + } + ret = glusterd_fetchspec_notify (THIS); + glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE); + volinfo->src_brick = volinfo->dst_brick = NULL; } - break; + break; case GF_REPLACE_OP_PAUSE: { gf_log ("", GF_LOG_DEBUG, "Recieved pause - doing nothing"); - ret = rb_do_operation_pause (volinfo, src_brickinfo, dst_brickinfo); - if (!glusterd_is_local_addr (dst_brickinfo->hostname)) { - gf_log ("", GF_LOG_NORMAL, - "I AM THE DESTINATION HOST"); - ret = rb_kill_destination_brick (volinfo, dst_brickinfo); + ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); + if (ctx) { + ret = rb_do_operation_pause (volinfo, src_brickinfo, dst_brickinfo); if (ret) { - gf_log ("", GF_LOG_DEBUG, - "Failed to kill destination brick"); + gf_log ("", GF_LOG_ERROR, + "Pause operation failed"); goto out; } } + + glusterd_set_rb_status (volinfo, GF_RB_STATUS_PAUSED); } break; case GF_REPLACE_OP_ABORT: { + ret = dict_set_int32 (volinfo->dict, "enable-pump", 0); if (ret) { gf_log ("", GF_LOG_CRITICAL, "Unable to disable pump"); } - ret = rb_do_operation_abort (volinfo, src_brickinfo, dst_brickinfo); if (!glusterd_is_local_addr (dst_brickinfo->hostname)) { gf_log ("", GF_LOG_NORMAL, "I AM THE DESTINATION HOST"); @@ -2946,15 +3096,32 @@ glusterd_op_replace_brick (gd1_mgmt_stage_op_req *req, dict_t *rsp_dict) } } + ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); + if (ctx) { + ret = rb_do_operation_abort (volinfo, src_brickinfo, dst_brickinfo); + if (ret) { + gf_log ("", GF_LOG_ERROR, + "Abort operation failed"); + goto out; + } + } + glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE); + volinfo->src_brick = volinfo->dst_brick = NULL; } - break; + break; case GF_REPLACE_OP_STATUS: { gf_log ("", GF_LOG_DEBUG, "received status - doing nothing"); - ret = rb_do_operation_status (volinfo, src_brickinfo, - dst_brickinfo); + ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); + if (ctx) { + ret = rb_do_operation_status (volinfo, src_brickinfo, + dst_brickinfo); + if (ret) + goto out; + } + } break; @@ -4015,21 +4182,6 @@ out: return ret; } -static gf_boolean_t -rb_check_brick_signin (glusterd_brickinfo_t *brickinfo) -{ - gf_boolean_t value; - - value = brickinfo->signed_in; - - if (value == _gf_true) { - gf_log ("", GF_LOG_DEBUG, - "Brick has signed in. Continuing..."); - } - - return value; -} - void glusterd_do_replace_brick (void *data) { @@ -4041,7 +4193,6 @@ glusterd_do_replace_brick (void *data) char *src_brick = NULL; char *dst_brick = NULL; char *volname = NULL; - gf_boolean_t brick_signin = _gf_false; glusterd_brickinfo_t *src_brickinfo = NULL; glusterd_brickinfo_t *dst_brickinfo = NULL; glusterd_conf_t *priv = NULL; @@ -4128,24 +4279,23 @@ glusterd_do_replace_brick (void *data) ret = dict_get_int32 (dict, "dst-brick-port", &dst_port); if (ret) { gf_log ("", GF_LOG_ERROR, "Unable to get dst-brick port"); - goto out; } dst_brickinfo->port = dst_port; src_brickinfo->port = src_port; - brick_signin = rb_check_brick_signin (src_brickinfo); - if (brick_signin == _gf_false) { - gf_log ("", GF_LOG_DEBUG, - "Marking replace brick to fail due to brick " - "not having signed-in in 10secs"); - ret = -1; - goto out; - } - switch (op) { case GF_REPLACE_OP_START: + if (!dst_port) { + ret = -1; + goto out; + } + ret = rb_do_operation_start (volinfo, src_brickinfo, dst_brickinfo); + if (ret) { + glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE); + goto out; + } break; case GF_REPLACE_OP_PAUSE: case GF_REPLACE_OP_ABORT: @@ -4541,6 +4691,7 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx) gd1_mgmt_stage_op_req *req = NULL; glusterd_op_stage_ctx_t *stage_ctx = NULL; int32_t status = 0; + dict_t *rsp_dict = NULL; char *op_errstr = NULL; GF_ASSERT (ctx); @@ -4549,19 +4700,32 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx) req = &stage_ctx->stage_req; - status = glusterd_op_stage_validate (req, &op_errstr); + + rsp_dict = dict_new (); + if (!rsp_dict) { + gf_log ("", GF_LOG_DEBUG, + "Out of memory"); + return -1; + } + + status = glusterd_op_stage_validate (req, &op_errstr, + rsp_dict); if (status) { gf_log ("", GF_LOG_ERROR, "Validate failed: %d", status); } - ret = glusterd_op_stage_send_resp (stage_ctx->req, req->op, status, op_errstr); + ret = glusterd_op_stage_send_resp (stage_ctx->req, req->op, + status, op_errstr, rsp_dict); if (op_errstr && (strcmp (op_errstr, ""))) GF_FREE (op_errstr); gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret); + if (rsp_dict) + dict_unref (rsp_dict); + return ret; } @@ -4630,7 +4794,8 @@ glusterd_op_sm_transition_state (glusterd_op_info_t *opinfo, } int32_t -glusterd_op_stage_validate (gd1_mgmt_stage_op_req *req, char **op_errstr) +glusterd_op_stage_validate (gd1_mgmt_stage_op_req *req, char **op_errstr, + dict_t *rsp_dict) { int ret = -1; @@ -4658,7 +4823,8 @@ glusterd_op_stage_validate (gd1_mgmt_stage_op_req *req, char **op_errstr) break; case GD_OP_REPLACE_BRICK: - ret = glusterd_op_stage_replace_brick (req, op_errstr); + ret = glusterd_op_stage_replace_brick (req, op_errstr, + rsp_dict); break; case GD_OP_SET_VOLUME: diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h index 440154bd071..775cab07c8e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h @@ -171,7 +171,8 @@ int glusterd_op_build_payload (glusterd_op_t op, gd1_mgmt_stage_op_req **req); int32_t -glusterd_op_stage_validate (gd1_mgmt_stage_op_req *req, char **op_errstr); +glusterd_op_stage_validate (gd1_mgmt_stage_op_req *req, char **op_errstr, + dict_t *rsp_dict); int32_t glusterd_op_commit_perform (gd1_mgmt_stage_op_req *req, char **op_errstr, diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c index f24ff85c9b9..934fc70307a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c +++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c @@ -408,6 +408,13 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req) goto out; } + if (glusterd_is_rb_started (volinfo) || + glusterd_is_rb_paused (volinfo)) { + gf_log ("glusterd", GF_LOG_DEBUG, + "Replace brick is in progress on volume %s", + cli_req.volname); + goto out; + } volinfo->defrag = GF_CALLOC (1, sizeof (glusterd_defrag_info_t), gf_gld_mt_defrag_info); if (!volinfo->defrag) diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 11a19d2a25e..cceecc11ce7 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -2337,3 +2337,53 @@ out: gf_log ("", GF_LOG_DEBUG, "returning %d ", ret); return ret; } + +inline int +glusterd_is_rb_started(glusterd_volinfo_t *volinfo) +{ + gf_log ("", GF_LOG_DEBUG, + "is_rb_started:status=%d", volinfo->rb_status); + return (volinfo->rb_status == GF_RB_STATUS_STARTED); + +} + +inline int +glusterd_is_rb_paused ( glusterd_volinfo_t *volinfo) +{ + gf_log ("", GF_LOG_DEBUG, + "is_rb_paused:status=%d", volinfo->rb_status); + + return (volinfo->rb_status == GF_RB_STATUS_PAUSED); +} + +inline int +glusterd_set_rb_status (glusterd_volinfo_t *volinfo, gf_rb_status_t status) +{ + gf_log ("", GF_LOG_DEBUG, + "setting status from %d to %d", + volinfo->rb_status, + status); + + volinfo->rb_status = status; + return 0; +} + +inline int +glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo, + glusterd_brickinfo_t *src, glusterd_brickinfo_t *dst) +{ + if (!volinfo->src_brick || !volinfo->dst_brick) + return -1; + + if (strcmp (volinfo->src_brick->hostname, src->hostname) || + strcmp (volinfo->src_brick->path, src->path)) { + gf_log("", GF_LOG_ERROR, "Replace brick src bricks differ"); + return -1; + } + if (strcmp (volinfo->dst_brick->hostname, dst->hostname) || + strcmp (volinfo->dst_brick->path, dst->path)) { + gf_log ("", GF_LOG_ERROR, "Replace brick dst bricks differ"); + return -1; + } + return 0; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index 79a15c02267..9db91fd5873 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -215,4 +215,16 @@ glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path, int glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path, glusterd_brickinfo_t **brickinfo); +int +glusterd_is_rb_started (glusterd_volinfo_t *volinfo); + +int +glusterd_is_rb_paused (glusterd_volinfo_t *volinfo); + +int +glusterd_set_rb_status (glusterd_volinfo_t *volinfo, gf_rb_status_t status); + +int +glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo, + glusterd_brickinfo_t *src_brick, glusterd_brickinfo_t *dst_brick); #endif diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 7763962cdd1..9d7401f99b5 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -151,6 +151,13 @@ typedef enum gf_transport_type_ { GF_TRANSPORT_RDMA, } gf_transport_type; + +typedef enum gf_rb_status_ { + GF_RB_STATUS_NONE, + GF_RB_STATUS_STARTED, + GF_RB_STATUS_PAUSED, +} gf_rb_status_t; + struct glusterd_volinfo_ { char volname[GLUSTERD_MAX_VOLUME_NAME]; int type; @@ -169,6 +176,11 @@ struct glusterd_volinfo_ { uint64_t lookedup_files; glusterd_defrag_info_t *defrag; + /* Replace brick status */ + gf_rb_status_t rb_status; + glusterd_brickinfo_t *src_brick; + glusterd_brickinfo_t *dst_brick; + int version; uint32_t cksum; gf_transport_type transport_type; @@ -291,7 +303,8 @@ glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status); int glusterd_op_stage_send_resp (rpcsvc_request_t *req, - int32_t op, int32_t status, char *op_errstr); + int32_t op, int32_t status, + char *op_errstr, dict_t *rsp_dict); int glusterd_op_commmit_send_resp (rpcsvc_request_t *req, diff --git a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c index cc2f84d380c..b7ffd0dd682 100644 --- a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c +++ b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c @@ -496,6 +496,63 @@ out: return ret; } +static int32_t +glusterd_rb_use_rsp_dict (dict_t *rsp_dict) +{ + int32_t src_port = 0; + int32_t dst_port = 0; + int ret = 0; + dict_t *ctx = NULL; + + + ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); + if (!ctx) { + gf_log ("", GF_LOG_ERROR, + "Operation Context is not present"); + GF_ASSERT (0); + } + + if (rsp_dict) { + ret = dict_get_int32 (rsp_dict, "src-brick-port", &src_port); + if (ret == 0) { + gf_log ("", GF_LOG_DEBUG, + "src-brick-port=%d found", src_port); + } + + ret = dict_get_int32 (rsp_dict, "dst-brick-port", &dst_port); + if (ret == 0) { + gf_log ("", GF_LOG_DEBUG, + "dst-brick-port=%d found", dst_port); + } + + } + + if (src_port) { + ret = dict_set_int32 (ctx, "src-brick-port", + src_port); + if (ret) { + gf_log ("", GF_LOG_DEBUG, + "Could not set src-brick"); + goto out; + } + } + + if (dst_port) { + ret = dict_set_int32 (ctx, "dst-brick-port", + dst_port); + if (ret) { + gf_log ("", GF_LOG_DEBUG, + "Could not set dst-brick"); + goto out; + } + + } + +out: + return ret; + +} + int32_t glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov, int count, void *myframe) @@ -506,6 +563,7 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov, glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE; glusterd_peerinfo_t *peerinfo = NULL; char str[50] = {0,}; + dict_t *dict = NULL; GF_ASSERT (req); @@ -526,6 +584,24 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov, } uuid_unparse (rsp.uuid, str); + if (rsp.dict.dict_len) { + /* Unserialize the dictionary */ + dict = dict_new (); + + ret = dict_unserialize (rsp.dict.dict_val, + rsp.dict.dict_len, + &dict); + if (ret < 0) { + gf_log ("glusterd", GF_LOG_ERROR, + "failed to " + "unserialize rsp-buffer to dictionary"); + event_type = GD_OP_EVENT_RCVD_RJT; + goto out; + } else { + dict->extra_stdfree = rsp.dict.dict_val; + } + } + op_ret = rsp.op_ret; gf_log ("glusterd", GF_LOG_NORMAL, @@ -552,6 +628,12 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov, event_type = GD_OP_EVENT_RCVD_ACC; } + switch (rsp.op) { + case GD_OP_REPLACE_BRICK: + glusterd_rb_use_rsp_dict (dict); + break; + } + ret = glusterd_op_sm_inject_event (event_type, NULL); if (!ret) { @@ -583,47 +665,6 @@ out: } -static int32_t -glusterd_rb_use_rsp_dict (dict_t *rsp_dict) -{ - int32_t src_port = 0; - int ret = 0; - dict_t *ctx = NULL; - - - if (rsp_dict) { - ret = dict_get_int32 (rsp_dict, "src-brick-port", &src_port); - if (ret) { - gf_log ("", GF_LOG_DEBUG, - "src-brick-port not present"); - ret = 0; - goto out; - } - - if (src_port) { - ctx = glusterd_op_get_ctx (GD_OP_REPLACE_BRICK); - if (!ctx) { - gf_log ("", GF_LOG_ERROR, - "Operation Context is not present"); - ret = 0; - goto out; - } - - ret = dict_set_int32 (ctx, "src-brick-port", - src_port); - if (ret) { - gf_log ("", GF_LOG_DEBUG, - "Could not set src-brick"); - goto out; - } - } - } - -out: - return ret; - -} - int32_t glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov, int count, void *myframe) @@ -643,6 +684,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov, rsp.op_ret = -1; rsp.op_errno = EINVAL; rsp.op_errstr = "error"; + event_type = GD_OP_EVENT_RCVD_RJT; goto out; } @@ -652,6 +694,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov, rsp.op_ret = -1; rsp.op_errno = EINVAL; rsp.op_errstr = "error"; + event_type = GD_OP_EVENT_RCVD_RJT; goto out; } uuid_unparse (rsp.uuid, str); @@ -667,6 +710,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov, gf_log ("glusterd", GF_LOG_ERROR, "failed to " "unserialize rsp-buffer to dictionary"); + event_type = GD_OP_EVENT_RCVD_RJT; goto out; } else { dict->extra_stdfree = rsp.dict.dict_val; @@ -696,6 +740,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov, goto out; } } else { + event_type = GD_OP_EVENT_RCVD_ACC; switch (rsp.op) { case GD_OP_REPLACE_BRICK: ret = glusterd_rb_use_rsp_dict (dict); @@ -710,9 +755,9 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov, default: break; } - event_type = GD_OP_EVENT_RCVD_ACC; } +out: ret = glusterd_op_sm_inject_event (event_type, NULL); if (!ret) { @@ -720,7 +765,6 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov, glusterd_op_sm (); } -out: if (dict) dict_unref (dict); if (rsp.op_errstr && strcmp (rsp.op_errstr, "error")) @@ -1158,7 +1202,8 @@ glusterd3_1_stage_op (call_frame_t *frame, xlator_t *this, if (ret) goto out; - ret = glusterd_op_stage_validate (req, &op_errstr); + /* rsp_dict NULL from source */ + ret = glusterd_op_stage_validate (req, &op_errstr, NULL); if (ret) { gf_log ("", GF_LOG_ERROR, "Staging failed"); -- cgit