diff options
47 files changed, 2813 insertions, 294 deletions
diff --git a/glusterfsd/src/glusterfsd-messages.h b/glusterfsd/src/glusterfsd-messages.h index 602cd9ecfd5..94312a5b1bb 100644 --- a/glusterfsd/src/glusterfsd-messages.h +++ b/glusterfsd/src/glusterfsd-messages.h @@ -35,6 +35,7 @@ GLFS_MSGID(GLUSTERFSD, glusterfsd_msg_1, glusterfsd_msg_2, glusterfsd_msg_3, glusterfsd_msg_28, glusterfsd_msg_29, glusterfsd_msg_30, glusterfsd_msg_31, glusterfsd_msg_32, glusterfsd_msg_33, glusterfsd_msg_34, glusterfsd_msg_35, glusterfsd_msg_36, - glusterfsd_msg_37, glusterfsd_msg_38, glusterfsd_msg_39); + glusterfsd_msg_37, glusterfsd_msg_38, glusterfsd_msg_39, + glusterfsd_msg_40, glusterfsd_msg_41, glusterfsd_msg_42); #endif /* !_GLUSTERFSD_MESSAGES_H_ */ diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c index 15acc109fff..1d2cd1ae6e1 100644 --- a/glusterfsd/src/glusterfsd-mgmt.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -48,7 +48,20 @@ int glusterfs_graph_unknown_options(glusterfs_graph_t *graph); int emancipate(glusterfs_ctx_t *ctx, int ret); +int +glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp, + char *volfile_id, char *checksum); +int +glusterfs_mux_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx, + gf_volfile_t *volfile_obj, char *checksum); +int +glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp, + char *volfile_id, char *checksum); +int +glusterfs_process_svc_detach(glusterfs_ctx_t *ctx, gf_volfile_t *volfile_obj); +gf_boolean_t +mgmt_is_multiplexed_daemon(char *name); int mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data) { @@ -62,6 +75,96 @@ mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data) } int +mgmt_process_volfile(const char *volfile, ssize_t size, char *volfile_id) +{ + glusterfs_ctx_t *ctx = NULL; + int ret = 0; + FILE *tmpfp = NULL; + gf_volfile_t *volfile_obj = NULL; + gf_volfile_t *volfile_tmp = NULL; + char sha256_hash[SHA256_DIGEST_LENGTH] = { + 0, + }; + int tmp_fd = -1; + char template[] = "/tmp/glfs.volfile.XXXXXX"; + + glusterfs_compute_sha256((const unsigned char *)volfile, size, sha256_hash); + ctx = THIS->ctx; + LOCK(&ctx->volfile_lock); + { + list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list) + { + if (!strcmp(volfile_id, volfile_obj->vol_id)) { + if (!memcmp(sha256_hash, volfile_obj->volfile_checksum, + sizeof(volfile_obj->volfile_checksum))) { + UNLOCK(&ctx->volfile_lock); + gf_msg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_40, + "No change in volfile, continuing"); + goto out; + } + volfile_tmp = volfile_obj; + break; + } + } + + /* coverity[secure_temp] mkstemp uses 0600 as the mode */ + tmp_fd = mkstemp(template); + if (-1 == tmp_fd) { + UNLOCK(&ctx->volfile_lock); + gf_msg(THIS->name, GF_LOG_ERROR, 0, glusterfsd_msg_39, + "Unable to create temporary file: %s", template); + ret = -1; + goto out; + } + + /* Calling unlink so that when the file is closed or program + * terminates the temporary file is deleted. + */ + ret = sys_unlink(template); + if (ret < 0) { + gf_msg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_39, + "Unable to delete temporary file: %s", template); + ret = 0; + } + + tmpfp = fdopen(tmp_fd, "w+b"); + if (!tmpfp) { + ret = -1; + goto unlock; + } + + fwrite(volfile, size, 1, tmpfp); + fflush(tmpfp); + if (ferror(tmpfp)) { + ret = -1; + goto unlock; + } + + if (!volfile_tmp) { + /* There is no checksum in the list, which means simple attach + * the volfile + */ + ret = glusterfs_process_svc_attach_volfp(ctx, tmpfp, volfile_id, + sha256_hash); + goto unlock; + } + ret = glusterfs_mux_volfile_reconfigure(tmpfp, ctx, volfile_obj, + sha256_hash); + if (ret < 0) { + gf_msg_debug("glusterfsd-mgmt", EINVAL, "Reconfigure failed !!"); + } + } +unlock: + UNLOCK(&ctx->volfile_lock); +out: + if (tmpfp) + fclose(tmpfp); + else if (tmp_fd != -1) + sys_close(tmp_fd); + return ret; +} + +int mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data) { return 0; @@ -966,6 +1069,110 @@ glusterfs_handle_attach(rpcsvc_request_t *req) } int +glusterfs_handle_svc_attach(rpcsvc_request_t *req) +{ + int32_t ret = -1; + gd1_mgmt_brick_op_req xlator_req = { + 0, + }; + xlator_t *this = NULL; + glusterfs_ctx_t *ctx = NULL; + + GF_ASSERT(req); + this = THIS; + GF_ASSERT(this); + + ctx = this->ctx; + ret = xdr_to_generic(req->msg[0], &xlator_req, + (xdrproc_t)xdr_gd1_mgmt_brick_op_req); + + if (ret < 0) { + /*failed to decode msg;*/ + req->rpc_err = GARBAGE_ARGS; + goto out; + } + gf_msg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_41, + "received attach " + "request for volfile-id=%s", + xlator_req.name); + ret = 0; + + if (ctx->active) { + ret = mgmt_process_volfile(xlator_req.input.input_val, + xlator_req.input.input_len, xlator_req.name); + } else { + gf_msg(this->name, GF_LOG_WARNING, EINVAL, glusterfsd_msg_42, + "got attach for %s but no active graph", xlator_req.name); + } +out: + if (xlator_req.input.input_val) + free(xlator_req.input.input_val); + if (xlator_req.name) + free(xlator_req.name); + glusterfs_translator_info_response_send(req, ret, NULL, NULL); + return 0; +} + +int +glusterfs_handle_svc_detach(rpcsvc_request_t *req) +{ + gd1_mgmt_brick_op_req xlator_req = { + 0, + }; + ssize_t ret; + glusterfs_ctx_t *ctx = NULL; + gf_volfile_t *volfile_obj = NULL; + gf_volfile_t *volfile_tmp = NULL; + + ret = xdr_to_generic(req->msg[0], &xlator_req, + (xdrproc_t)xdr_gd1_mgmt_brick_op_req); + if (ret < 0) { + req->rpc_err = GARBAGE_ARGS; + return -1; + } + ctx = glusterfsd_ctx; + + LOCK(&ctx->volfile_lock); + { + list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list) + { + if (!strcmp(xlator_req.name, volfile_obj->vol_id)) { + volfile_tmp = volfile_obj; + break; + } + } + + if (!volfile_tmp) { + UNLOCK(&ctx->volfile_lock); + gf_msg(THIS->name, GF_LOG_ERROR, 0, glusterfsd_msg_41, + "can't detach %s - not found", xlator_req.name); + /* + * Used to be -ENOENT. However, the caller asked us to + * make sure it's down and if it's already down that's + * good enough. + */ + ret = 0; + goto out; + } + ret = glusterfs_process_svc_detach(ctx, volfile_tmp); + if (ret) { + UNLOCK(&ctx->volfile_lock); + gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, EINVAL, glusterfsd_msg_41, + "Could not detach " + "old graph. Aborting the reconfiguration operation"); + goto out; + } + } + UNLOCK(&ctx->volfile_lock); +out: + glusterfs_terminate_response_send(req, ret); + free(xlator_req.name); + xlator_req.name = NULL; + + return 0; +} + +int glusterfs_handle_dump_metrics(rpcsvc_request_t *req) { int32_t ret = -1; @@ -1849,6 +2056,13 @@ rpcsvc_actor_t glusterfs_actors[GLUSTERD_BRICK_MAXVALUE] = { [GLUSTERD_DUMP_METRICS] = {"DUMP METRICS", GLUSTERD_DUMP_METRICS, glusterfs_handle_dump_metrics, NULL, 0, DRC_NA}, + + [GLUSTERD_SVC_ATTACH] = {"ATTACH CLIENT", GLUSTERD_SVC_ATTACH, + glusterfs_handle_svc_attach, NULL, 0, DRC_NA}, + + [GLUSTERD_SVC_DETACH] = {"DETACH CLIENT", GLUSTERD_SVC_DETACH, + glusterfs_handle_svc_detach, NULL, 0, DRC_NA}, + }; struct rpcsvc_program glusterfs_mop_prog = { @@ -1996,14 +2210,17 @@ mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count, } volfile: - ret = 0; size = rsp.op_ret; + volfile_id = frame->local; + if (mgmt_is_multiplexed_daemon(ctx->cmd_args.process_name)) { + ret = mgmt_process_volfile((const char *)rsp.spec, size, volfile_id); + goto post_graph_mgmt; + } + ret = 0; glusterfs_compute_sha256((const unsigned char *)rsp.spec, size, sha256_hash); - volfile_id = frame->local; - LOCK(&ctx->volfile_lock); { locked = 1; @@ -2105,6 +2322,7 @@ volfile: } INIT_LIST_HEAD(&volfile_tmp->volfile_list); + volfile_tmp->graph = ctx->active; list_add(&volfile_tmp->volfile_list, &ctx->volfile_list); snprintf(volfile_tmp->vol_id, sizeof(volfile_tmp->vol_id), "%s", volfile_id); @@ -2116,6 +2334,7 @@ volfile: locked = 0; +post_graph_mgmt: if (!is_mgmt_rpc_reconnect) { need_emancipate = 1; glusterfs_mgmt_pmap_signin(ctx); @@ -2269,10 +2488,21 @@ glusterfs_volfile_fetch(glusterfs_ctx_t *ctx) { xlator_t *server_xl = NULL; xlator_list_t *trav; - int ret; + gf_volfile_t *volfile_obj = NULL; + int ret = 0; LOCK(&ctx->volfile_lock); { + if (ctx->active && + mgmt_is_multiplexed_daemon(ctx->cmd_args.process_name)) { + list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list) + { + ret |= glusterfs_volfile_fetch_one(ctx, volfile_obj->vol_id); + } + UNLOCK(&ctx->volfile_lock); + return ret; + } + if (ctx->active) { server_xl = ctx->active->first; if (strcmp(server_xl->type, "protocol/server") != 0) { diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index f00c2fd0c32..c4fb5dfc7e2 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -2637,24 +2637,6 @@ out: #endif int -glusterfs_graph_fini(glusterfs_graph_t *graph) -{ - xlator_t *trav = NULL; - - trav = graph->first; - - while (trav) { - if (trav->init_succeeded) { - trav->fini(trav); - trav->init_succeeded = 0; - } - trav = trav->next; - } - - return 0; -} - -int glusterfs_process_volfp(glusterfs_ctx_t *ctx, FILE *fp) { glusterfs_graph_t *graph = NULL; diff --git a/libglusterfs/src/defaults-tmpl.c b/libglusterfs/src/defaults-tmpl.c index 5bf64e8c6c6..82e7f78d7f3 100644 --- a/libglusterfs/src/defaults-tmpl.c +++ b/libglusterfs/src/defaults-tmpl.c @@ -127,6 +127,12 @@ default_notify(xlator_t *this, int32_t event, void *data, ...) GF_UNUSED int ret = 0; xlator_t *victim = data; + glusterfs_graph_t *graph = NULL; + + GF_VALIDATE_OR_GOTO("notify", this, out); + graph = this->graph; + GF_VALIDATE_OR_GOTO(this->name, graph, out); + switch (event) { case GF_EVENT_PARENT_UP: case GF_EVENT_PARENT_DOWN: { @@ -159,6 +165,17 @@ default_notify(xlator_t *this, int32_t event, void *data, ...) xlator_notify(parent->xlator, event, this, NULL); parent = parent->next; } + + if (event == GF_EVENT_CHILD_DOWN && + !(this->ctx && this->ctx->master) && (graph->top == this)) { + /* Make sure this is not a daemon with master xlator */ + pthread_mutex_lock(&graph->mutex); + { + graph->used = 0; + pthread_cond_broadcast(&graph->child_down_cond); + } + pthread_mutex_unlock(&graph->mutex); + } } break; case GF_EVENT_UPCALL: { xlator_list_t *parent = this->parents; @@ -205,7 +222,7 @@ default_notify(xlator_t *this, int32_t event, void *data, ...) * nothing to do with readability. */ } - +out: return 0; } diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h index 9d140c18ee3..ebec8898cf1 100644 --- a/libglusterfs/src/glusterfs/glusterfs.h +++ b/libglusterfs/src/glusterfs/glusterfs.h @@ -592,6 +592,10 @@ struct _glusterfs_graph { int used; /* Should be set when fuse gets first CHILD_UP */ uint32_t volfile_checksum; + void *last_xl; /* Stores the last xl of the graph, as of now only populated + in client multiplexed code path */ + pthread_mutex_t mutex; + pthread_cond_t child_down_cond; /* for broadcasting CHILD_DOWN */ }; typedef struct _glusterfs_graph glusterfs_graph_t; @@ -734,6 +738,7 @@ typedef struct { char volfile_checksum[SHA256_DIGEST_LENGTH]; char vol_id[NAME_MAX + 1]; struct list_head volfile_list; + glusterfs_graph_t *graph; } gf_volfile_t; @@ -817,4 +822,6 @@ gf_free_mig_locks(lock_migration_info_t *locks); int glusterfs_read_secure_access_file(void); +int +glusterfs_graph_fini(glusterfs_graph_t *graph); #endif /* _GLUSTERFS_H */ diff --git a/libglusterfs/src/glusterfs/libglusterfs-messages.h b/libglusterfs/src/glusterfs/libglusterfs-messages.h index e17e33e06fb..ea2aa606470 100644 --- a/libglusterfs/src/glusterfs/libglusterfs-messages.h +++ b/libglusterfs/src/glusterfs/libglusterfs-messages.h @@ -109,6 +109,8 @@ GLFS_MSGID( LG_MSG_PTHREAD_ATTR_INIT_FAILED, LG_MSG_INVALID_INODE_LIST, LG_MSG_COMPACT_FAILED, LG_MSG_COMPACT_STATUS, LG_MSG_UTIMENSAT_FAILED, LG_MSG_PTHREAD_NAMING_FAILED, LG_MSG_SYSCALL_RETURNS_WRONG, - LG_MSG_XXH64_TO_GFID_FAILED, LG_MSG_ASYNC_WARNING, LG_MSG_ASYNC_FAILURE); + LG_MSG_XXH64_TO_GFID_FAILED, LG_MSG_ASYNC_WARNING, LG_MSG_ASYNC_FAILURE, + LG_MSG_GRAPH_CLEANUP_FAILED, LG_MSG_GRAPH_SETUP_FAILED, + LG_MSG_GRAPH_DETACH_STARTED, LG_MSG_GRAPH_ATTACH_FAILED); #endif /* !_LG_MESSAGES_H_ */ diff --git a/libglusterfs/src/glusterfs/xlator.h b/libglusterfs/src/glusterfs/xlator.h index acd8be0913a..90396f93431 100644 --- a/libglusterfs/src/glusterfs/xlator.h +++ b/libglusterfs/src/glusterfs/xlator.h @@ -1087,4 +1087,7 @@ handle_default_options(xlator_t *xl, dict_t *options); void gluster_graph_take_reference(xlator_t *tree); + +gf_boolean_t +mgmt_is_multiplexed_daemon(char *name); #endif /* _XLATOR_H */ diff --git a/libglusterfs/src/graph.c b/libglusterfs/src/graph.c index bb5e67a5e60..a492dd891fd 100644 --- a/libglusterfs/src/graph.c +++ b/libglusterfs/src/graph.c @@ -114,6 +114,53 @@ out: return cert_depth; } +xlator_t * +glusterfs_get_last_xlator(glusterfs_graph_t *graph) +{ + xlator_t *trav = graph->first; + if (!trav) + return NULL; + + while (trav->next) + trav = trav->next; + + return trav; +} + +xlator_t * +glusterfs_mux_xlator_unlink(xlator_t *pxl, xlator_t *cxl) +{ + xlator_list_t *unlink = NULL; + xlator_list_t *prev = NULL; + xlator_list_t **tmp = NULL; + xlator_t *next_child = NULL; + xlator_t *xl = NULL; + + for (tmp = &pxl->children; *tmp; tmp = &(*tmp)->next) { + if ((*tmp)->xlator == cxl) { + unlink = *tmp; + *tmp = (*tmp)->next; + if (*tmp) + next_child = (*tmp)->xlator; + break; + } + prev = *tmp; + } + + if (!prev) + xl = pxl; + else if (prev->xlator) + xl = prev->xlator->graph->last_xl; + + if (xl) + xl->next = next_child; + if (next_child) + next_child->prev = xl; + + GF_FREE(unlink); + return next_child; +} + int glusterfs_xlator_link(xlator_t *pxl, xlator_t *cxl) { @@ -1092,6 +1139,8 @@ glusterfs_graph_destroy_residual(glusterfs_graph_t *graph) ret = xlator_tree_free_memacct(graph->first); list_del_init(&graph->list); + pthread_mutex_destroy(&graph->mutex); + pthread_cond_destroy(&graph->child_down_cond); GF_FREE(graph); return ret; @@ -1134,6 +1183,25 @@ out: } int +glusterfs_graph_fini(glusterfs_graph_t *graph) +{ + xlator_t *trav = NULL; + + trav = graph->first; + + while (trav) { + if (trav->init_succeeded) { + trav->cleanup_starting = 1; + trav->fini(trav); + trav->init_succeeded = 0; + } + trav = trav->next; + } + + return 0; +} + +int glusterfs_graph_attach(glusterfs_graph_t *orig_graph, char *path, glusterfs_graph_t **newgraph) { @@ -1256,3 +1324,386 @@ glusterfs_graph_attach(glusterfs_graph_t *orig_graph, char *path, return 0; } +int +glusterfs_muxsvc_cleanup_parent(glusterfs_ctx_t *ctx, + glusterfs_graph_t *parent_graph) +{ + if (parent_graph) { + if (parent_graph->first) { + xlator_destroy(parent_graph->first); + } + ctx->active = NULL; + GF_FREE(parent_graph); + parent_graph = NULL; + } + return 0; +} + +void * +glusterfs_graph_cleanup(void *arg) +{ + glusterfs_graph_t *graph = NULL; + glusterfs_ctx_t *ctx = THIS->ctx; + int ret = -1; + graph = arg; + + if (!graph) + return NULL; + + /* To destroy the graph, fitst sent a GF_EVENT_PARENT_DOWN + * Then wait for GF_EVENT_CHILD_DOWN to get on the top + * xl. Once we have GF_EVENT_CHILD_DOWN event, then proceed + * to fini. + * + * During fini call, this will take a last unref on rpc and + * rpc_transport_object. + */ + if (graph->first) + default_notify(graph->first, GF_EVENT_PARENT_DOWN, graph->first); + + ret = pthread_mutex_lock(&graph->mutex); + if (ret != 0) { + gf_msg("glusterfs", GF_LOG_ERROR, EAGAIN, LG_MSG_GRAPH_CLEANUP_FAILED, + "Failed to aquire a lock"); + goto out; + } + /* check and wait for CHILD_DOWN for top xlator*/ + while (graph->used) { + ret = pthread_cond_wait(&graph->child_down_cond, &graph->mutex); + if (ret != 0) + gf_msg("glusterfs", GF_LOG_INFO, 0, LG_MSG_GRAPH_CLEANUP_FAILED, + "cond wait failed "); + } + + ret = pthread_mutex_unlock(&graph->mutex); + if (ret != 0) { + gf_msg("glusterfs", GF_LOG_ERROR, EAGAIN, LG_MSG_GRAPH_CLEANUP_FAILED, + "Failed to release a lock"); + } + + /* Though we got a child down on top xlator, we have to wait until + * all the notifier to exit. Because there should not be any threads + * that access xl variables. + */ + pthread_mutex_lock(&ctx->notify_lock); + { + while (ctx->notifying) + pthread_cond_wait(&ctx->notify_cond, &ctx->notify_lock); + } + pthread_mutex_unlock(&ctx->notify_lock); + + glusterfs_graph_fini(graph); + glusterfs_graph_destroy(graph); +out: + return NULL; +} + +glusterfs_graph_t * +glusterfs_muxsvc_setup_parent_graph(glusterfs_ctx_t *ctx, char *name, + char *type) +{ + glusterfs_graph_t *parent_graph = NULL; + xlator_t *ixl = NULL; + int ret = -1; + parent_graph = GF_CALLOC(1, sizeof(*parent_graph), + gf_common_mt_glusterfs_graph_t); + if (!parent_graph) + goto out; + + INIT_LIST_HEAD(&parent_graph->list); + + ctx->active = parent_graph; + ixl = GF_CALLOC(1, sizeof(*ixl), gf_common_mt_xlator_t); + if (!ixl) + goto out; + + ixl->ctx = ctx; + ixl->graph = parent_graph; + ixl->options = dict_new(); + if (!ixl->options) + goto out; + + ixl->name = gf_strdup(name); + if (!ixl->name) + goto out; + + ixl->is_autoloaded = 1; + + if (xlator_set_type(ixl, type) == -1) { + gf_msg("glusterfs", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_SETUP_FAILED, + "%s (%s) set type failed", name, type); + goto out; + } + + glusterfs_graph_set_first(parent_graph, ixl); + parent_graph->top = ixl; + ixl = NULL; + + gettimeofday(&parent_graph->dob, NULL); + fill_uuid(parent_graph->graph_uuid, 128); + parent_graph->id = ctx->graph_id++; + ret = 0; +out: + if (ixl) + xlator_destroy(ixl); + + if (ret) { + glusterfs_muxsvc_cleanup_parent(ctx, parent_graph); + parent_graph = NULL; + } + return parent_graph; +} + +int +glusterfs_process_svc_detach(glusterfs_ctx_t *ctx, gf_volfile_t *volfile_obj) +{ + xlator_t *last_xl = NULL; + glusterfs_graph_t *graph = NULL; + glusterfs_graph_t *parent_graph = NULL; + pthread_t clean_graph = { + 0, + }; + int ret = -1; + xlator_t *xl = NULL; + + if (!ctx || !ctx->active || !volfile_obj) + goto out; + parent_graph = ctx->active; + graph = volfile_obj->graph; + if (graph && graph->first) + xl = graph->first; + + last_xl = graph->last_xl; + if (last_xl) + last_xl->next = NULL; + if (!xl || xl->cleanup_starting) + goto out; + + xl->cleanup_starting = 1; + gf_msg("mgmt", GF_LOG_INFO, 0, LG_MSG_GRAPH_DETACH_STARTED, + "detaching child %s", volfile_obj->vol_id); + + list_del_init(&volfile_obj->volfile_list); + glusterfs_mux_xlator_unlink(parent_graph->top, xl); + parent_graph->last_xl = glusterfs_get_last_xlator(parent_graph); + parent_graph->xl_count -= graph->xl_count; + parent_graph->leaf_count -= graph->leaf_count; + default_notify(xl, GF_EVENT_PARENT_DOWN, xl); + parent_graph->id++; + ret = 0; +out: + if (!ret) { + list_del_init(&volfile_obj->volfile_list); + if (graph) { + ret = gf_thread_create_detached( + &clean_graph, glusterfs_graph_cleanup, graph, "graph_clean"); + if (ret) { + gf_msg("glusterfs", GF_LOG_ERROR, EINVAL, + LG_MSG_GRAPH_CLEANUP_FAILED, + "%s failed to create clean " + "up thread", + volfile_obj->vol_id); + ret = 0; + } + } + GF_FREE(volfile_obj); + } + return ret; +} + +int +glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp, + char *volfile_id, char *checksum) +{ + glusterfs_graph_t *graph = NULL; + glusterfs_graph_t *parent_graph = NULL; + glusterfs_graph_t *clean_graph = NULL; + int ret = -1; + xlator_t *xl = NULL; + xlator_t *last_xl = NULL; + gf_volfile_t *volfile_obj = NULL; + pthread_t thread_id = { + 0, + }; + + if (!ctx) + goto out; + parent_graph = ctx->active; + graph = glusterfs_graph_construct(fp); + if (!graph) { + gf_msg("glusterfsd", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_ATTACH_FAILED, + "failed to construct the graph"); + goto out; + } + graph->last_xl = glusterfs_get_last_xlator(graph); + + for (xl = graph->first; xl; xl = xl->next) { + if (strcmp(xl->type, "mount/fuse") == 0) { + gf_msg("glusterfsd", GF_LOG_ERROR, EINVAL, + LG_MSG_GRAPH_ATTACH_FAILED, + "fuse xlator cannot be specified in volume file"); + goto out; + } + } + + graph->leaf_count = glusterfs_count_leaves(glusterfs_root(graph)); + xl = graph->first; + /* TODO memory leaks everywhere need to free graph in case of error */ + if (glusterfs_graph_prepare(graph, ctx, xl->name)) { + gf_msg("glusterfsd", GF_LOG_WARNING, EINVAL, LG_MSG_GRAPH_ATTACH_FAILED, + "failed to prepare graph for xlator %s", xl->name); + ret = -1; + goto out; + } else if (glusterfs_graph_init(graph)) { + gf_msg("glusterfsd", GF_LOG_WARNING, EINVAL, LG_MSG_GRAPH_ATTACH_FAILED, + "failed to initialize graph for xlator %s", xl->name); + ret = -1; + goto out; + } else if (glusterfs_graph_parent_up(graph)) { + gf_msg("glusterfsd", GF_LOG_WARNING, EINVAL, LG_MSG_GRAPH_ATTACH_FAILED, + "failed to link the graphs for xlator %s ", xl->name); + ret = -1; + goto out; + } + + if (!parent_graph) { + parent_graph = glusterfs_muxsvc_setup_parent_graph(ctx, "glustershd", + "debug/io-stats"); + if (!parent_graph) + goto out; + ((xlator_t *)parent_graph->top)->next = xl; + clean_graph = parent_graph; + } else { + last_xl = parent_graph->last_xl; + if (last_xl) + last_xl->next = xl; + xl->prev = last_xl; + } + parent_graph->last_xl = graph->last_xl; + + ret = glusterfs_xlator_link(parent_graph->top, xl); + if (ret) { + gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_EVENT_NOTIFY_FAILED, + "parent up notification failed"); + goto out; + } + parent_graph->xl_count += graph->xl_count; + parent_graph->leaf_count += graph->leaf_count; + parent_graph->id++; + + if (!volfile_obj) { + volfile_obj = GF_CALLOC(1, sizeof(gf_volfile_t), gf_common_volfile_t); + if (!volfile_obj) { + ret = -1; + goto out; + } + } + + graph->used = 1; + parent_graph->id++; + list_add(&graph->list, &ctx->graphs); + INIT_LIST_HEAD(&volfile_obj->volfile_list); + volfile_obj->graph = graph; + snprintf(volfile_obj->vol_id, sizeof(volfile_obj->vol_id), "%s", + volfile_id); + memcpy(volfile_obj->volfile_checksum, checksum, + sizeof(volfile_obj->volfile_checksum)); + list_add_tail(&volfile_obj->volfile_list, &ctx->volfile_list); + + gf_log_dump_graph(fp, graph); + graph = NULL; + + ret = 0; +out: + if (ret) { + if (graph) { + gluster_graph_take_reference(graph->first); + ret = gf_thread_create_detached(&thread_id, glusterfs_graph_cleanup, + graph, "graph_clean"); + if (ret) { + gf_msg("glusterfs", GF_LOG_ERROR, EINVAL, + LG_MSG_GRAPH_CLEANUP_FAILED, + "%s failed to create clean " + "up thread", + volfile_id); + ret = 0; + } + } + if (clean_graph) + glusterfs_muxsvc_cleanup_parent(ctx, clean_graph); + } + return ret; +} + +int +glusterfs_mux_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx, + gf_volfile_t *volfile_obj, char *checksum) +{ + glusterfs_graph_t *oldvolfile_graph = NULL; + glusterfs_graph_t *newvolfile_graph = NULL; + + int ret = -1; + + if (!ctx) { + gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, 0, LG_MSG_CTX_NULL, + "ctx is NULL"); + goto out; + } + + /* Change the message id */ + if (!volfile_obj) { + gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, 0, LG_MSG_CTX_NULL, + "failed to get volfile object"); + goto out; + } + + oldvolfile_graph = volfile_obj->graph; + if (!oldvolfile_graph) { + goto out; + } + + newvolfile_graph = glusterfs_graph_construct(newvolfile_fp); + + if (!newvolfile_graph) { + goto out; + } + newvolfile_graph->last_xl = glusterfs_get_last_xlator(newvolfile_graph); + + glusterfs_graph_prepare(newvolfile_graph, ctx, newvolfile_graph->first); + + if (!is_graph_topology_equal(oldvolfile_graph, newvolfile_graph)) { + ret = glusterfs_process_svc_detach(ctx, volfile_obj); + if (ret) { + gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, EINVAL, + LG_MSG_GRAPH_CLEANUP_FAILED, + "Could not detach " + "old graph. Aborting the reconfiguration operation"); + goto out; + } + ret = glusterfs_process_svc_attach_volfp(ctx, newvolfile_fp, + volfile_obj->vol_id, checksum); + goto out; + } + + gf_msg_debug("glusterfsd-mgmt", 0, + "Only options have changed in the" + " new graph"); + + ret = glusterfs_graph_reconfigure(oldvolfile_graph, newvolfile_graph); + if (ret) { + gf_msg_debug("glusterfsd-mgmt", 0, + "Could not reconfigure " + "new options in old graph"); + goto out; + } + memcpy(volfile_obj->volfile_checksum, checksum, + sizeof(volfile_obj->volfile_checksum)); + + ret = 0; +out: + + if (newvolfile_graph) + glusterfs_graph_destroy(newvolfile_graph); + + return ret; +} diff --git a/libglusterfs/src/graph.y b/libglusterfs/src/graph.y index 5b929856c5f..c60ff388b2a 100644 --- a/libglusterfs/src/graph.y +++ b/libglusterfs/src/graph.y @@ -542,6 +542,9 @@ glusterfs_graph_new () INIT_LIST_HEAD (&graph->list); + pthread_mutex_init(&graph->mutex, NULL); + pthread_cond_init(&graph->child_down_cond, NULL); + gettimeofday (&graph->dob, NULL); return graph; diff --git a/libglusterfs/src/libglusterfs.sym b/libglusterfs/src/libglusterfs.sym index 0d1142dea76..0fdb62c810f 100644 --- a/libglusterfs/src/libglusterfs.sym +++ b/libglusterfs/src/libglusterfs.sym @@ -1162,3 +1162,8 @@ gf_changelog_register_generic gf_gfid_generate_from_xxh64 find_xlator_option_in_cmd_args_t gf_d_type_from_ia_type +glusterfs_graph_fini +glusterfs_process_svc_attach_volfp +glusterfs_mux_volfile_reconfigure +glusterfs_process_svc_detach +mgmt_is_multiplexed_daemon diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c index d18b50facf4..0cf80c0f87c 100644 --- a/libglusterfs/src/statedump.c +++ b/libglusterfs/src/statedump.c @@ -810,7 +810,8 @@ gf_proc_dump_info(int signum, glusterfs_ctx_t *ctx) if (!ctx) goto out; - if (ctx && ctx->active) { + if (!mgmt_is_multiplexed_daemon(ctx->cmd_args.process_name) && + (ctx && ctx->active)) { top = ctx->active->first; for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) { brick_count++; diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c index 9bdbc569ab4..6bba99a74ef 100644 --- a/libglusterfs/src/xlator.c +++ b/libglusterfs/src/xlator.c @@ -1485,3 +1485,19 @@ gluster_graph_take_reference(xlator_t *tree) } return; } + +gf_boolean_t +mgmt_is_multiplexed_daemon(char *name) +{ + const char *mux_daemons[] = {"glustershd", NULL}; + int i; + + if (!name) + return _gf_false; + + for (i = 0; mux_daemons[i]; i++) { + if (!strcmp(name, mux_daemons[i])) + return _gf_true; + } + return _gf_false; +} diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h index 779878f52be..7275d7568b6 100644 --- a/rpc/rpc-lib/src/protocol-common.h +++ b/rpc/rpc-lib/src/protocol-common.h @@ -245,6 +245,8 @@ enum glusterd_brick_procnum { GLUSTERD_NODE_BITROT, GLUSTERD_BRICK_ATTACH, GLUSTERD_DUMP_METRICS, + GLUSTERD_SVC_ATTACH, + GLUSTERD_SVC_DETACH, GLUSTERD_BRICK_MAXVALUE, }; diff --git a/tests/basic/glusterd/heald.t b/tests/basic/glusterd/heald.t index ca112ad0b75..7dae3c3f0fb 100644 --- a/tests/basic/glusterd/heald.t +++ b/tests/basic/glusterd/heald.t @@ -7,11 +7,16 @@ # Covers enable/disable at the moment. Will be enhanced later to include # the other commands as well. +function is_pid_running { + local pid=$1 + num=`ps auxww | grep glustershd | grep $pid | grep -v grep | wc -l` + echo $num +} + cleanup; TEST glusterd TEST pidof glusterd -volfile=$(gluster system:: getwd)"/glustershd/glustershd-server.vol" #Commands should fail when volume doesn't exist TEST ! $CLI volume heal non-existent-volume enable TEST ! $CLI volume heal non-existent-volume disable @@ -20,51 +25,55 @@ TEST ! $CLI volume heal non-existent-volume disable # volumes TEST $CLI volume create dist $H0:$B0/dist TEST $CLI volume start dist -TEST "[ -z $(get_shd_process_pid)]" +TEST "[ -z $(get_shd_process_pid dist)]" TEST ! $CLI volume heal dist enable TEST ! $CLI volume heal dist disable # Commands should work on replicate/disperse volume. TEST $CLI volume create r2 replica 2 $H0:$B0/r2_0 $H0:$B0/r2_1 -TEST "[ -z $(get_shd_process_pid)]" +TEST "[ -z $(get_shd_process_pid r2)]" TEST $CLI volume start r2 -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid r2 TEST $CLI volume heal r2 enable EXPECT "enable" volume_option r2 "cluster.self-heal-daemon" -EXPECT "enable" volgen_volume_option $volfile r2-replicate-0 cluster replicate self-heal-daemon -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid +volfiler2=$(gluster system:: getwd)"/vols/r2/r2-shd.vol" +EXPECT "enable" volgen_volume_option $volfiler2 r2-replicate-0 cluster replicate self-heal-daemon +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid r2 +pid=$( get_shd_process_pid r2 ) TEST $CLI volume heal r2 disable EXPECT "disable" volume_option r2 "cluster.self-heal-daemon" -EXPECT "disable" volgen_volume_option $volfile r2-replicate-0 cluster replicate self-heal-daemon -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid +EXPECT "disable" volgen_volume_option $volfiler2 r2-replicate-0 cluster replicate self-heal-daemon +EXPECT "1" is_pid_running $pid # Commands should work on disperse volume. TEST $CLI volume create ec2 disperse 3 redundancy 1 $H0:$B0/ec2_0 $H0:$B0/ec2_1 $H0:$B0/ec2_2 TEST $CLI volume start ec2 -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid ec2 TEST $CLI volume heal ec2 enable EXPECT "enable" volume_option ec2 "cluster.disperse-self-heal-daemon" -EXPECT "enable" volgen_volume_option $volfile ec2-disperse-0 cluster disperse self-heal-daemon -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid +volfileec2=$(gluster system:: getwd)"/vols/ec2/ec2-shd.vol" +EXPECT "enable" volgen_volume_option $volfileec2 ec2-disperse-0 cluster disperse self-heal-daemon +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid ec2 +pid=$(get_shd_process_pid ec2) TEST $CLI volume heal ec2 disable EXPECT "disable" volume_option ec2 "cluster.disperse-self-heal-daemon" -EXPECT "disable" volgen_volume_option $volfile ec2-disperse-0 cluster disperse self-heal-daemon -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "[0-9][0-9]*" get_shd_process_pid +EXPECT "disable" volgen_volume_option $volfileec2 ec2-disperse-0 cluster disperse self-heal-daemon +EXPECT "1" is_pid_running $pid #Check that shd graph is rewritten correctly on volume stop/start -EXPECT "Y" volgen_volume_exists $volfile ec2-disperse-0 cluster disperse -EXPECT "Y" volgen_volume_exists $volfile r2-replicate-0 cluster replicate +EXPECT "Y" volgen_volume_exists $volfileec2 ec2-disperse-0 cluster disperse + +EXPECT "Y" volgen_volume_exists $volfiler2 r2-replicate-0 cluster replicate TEST $CLI volume stop r2 -EXPECT "Y" volgen_volume_exists $volfile ec2-disperse-0 cluster disperse -EXPECT "N" volgen_volume_exists $volfile r2-replicate-0 cluster replicate +EXPECT "Y" volgen_volume_exists $volfileec2 ec2-disperse-0 cluster disperse TEST $CLI volume stop ec2 # When both the volumes are stopped glustershd volfile is not modified just the # process is stopped -TEST "[ -z $(get_shd_process_pid) ]" +TEST "[ -z $(get_shd_process_pid dist) ]" +TEST "[ -z $(get_shd_process_pid ec2) ]" TEST $CLI volume start r2 -EXPECT "N" volgen_volume_exists $volfile ec2-disperse-0 cluster disperse -EXPECT "Y" volgen_volume_exists $volfile r2-replicate-0 cluster replicate +EXPECT "Y" volgen_volume_exists $volfiler2 r2-replicate-0 cluster replicate TEST $CLI volume set r2 self-heal-daemon on TEST $CLI volume set r2 cluster.self-heal-daemon off diff --git a/tests/bugs/glusterd/reset-brick-and-daemons-follow-quorum.t b/tests/bugs/glusterd/reset-brick-and-daemons-follow-quorum.t index cdb1a3399c9..e6e65c48456 100644 --- a/tests/bugs/glusterd/reset-brick-and-daemons-follow-quorum.t +++ b/tests/bugs/glusterd/reset-brick-and-daemons-follow-quorum.t @@ -55,9 +55,9 @@ TEST kill_glusterd 1 #Bring back 1st glusterd TEST $glusterd_1 -# We need to wait till PROCESS_UP_TIMEOUT and then check shd service does not -# come up on node 2 -sleep $PROCESS_UP_TIMEOUT -EXPECT "N" shd_up_status_2 +# We need to wait till PROCESS_UP_TIMEOUT and then check shd service started +#on node 2, because once glusterd regains quorum, it will restart all volume +#level daemons +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" shd_up_status_2 cleanup; diff --git a/tests/volume.rc b/tests/volume.rc index fe6074b3cd6..1adc5b15c35 100644 --- a/tests/volume.rc +++ b/tests/volume.rc @@ -237,11 +237,13 @@ function ec_child_up_count_shd { } function get_shd_process_pid { - ps auxww | grep glusterfs | grep -E "glustershd/glustershd.pid" | awk '{print $2}' | head -1 + local vol=$1 + ps auxww | grep "process-name\ glustershd" | awk '{print $2}' | head -1 } function generate_shd_statedump { - generate_statedump $(get_shd_process_pid) + local vol=$1 + generate_statedump $(get_shd_process_pid $vol) } function generate_nfs_statedump { diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am index 6d09e37477c..1f7a3125ac2 100644 --- a/xlators/mgmt/glusterd/src/Makefile.am +++ b/xlators/mgmt/glusterd/src/Makefile.am @@ -18,11 +18,12 @@ glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \ glusterd-locks.c glusterd-snapshot.c glusterd-mgmt-handler.c \ glusterd-mgmt.c glusterd-peer-utils.c glusterd-statedump.c \ glusterd-snapshot-utils.c glusterd-conn-mgmt.c \ - glusterd-proc-mgmt.c glusterd-svc-mgmt.c glusterd-shd-svc.c \ + glusterd-proc-mgmt.c glusterd-svc-mgmt.c \ glusterd-nfs-svc.c glusterd-quotad-svc.c glusterd-svc-helper.c \ glusterd-conn-helper.c glusterd-snapd-svc.c glusterd-snapd-svc-helper.c \ glusterd-bitd-svc.c glusterd-scrub-svc.c glusterd-server-quorum.c \ glusterd-reset-brick.c glusterd-tierd-svc.c glusterd-tierd-svc-helper.c \ + glusterd-shd-svc.c glusterd-shd-svc-helper.c \ glusterd-gfproxyd-svc.c glusterd-gfproxyd-svc-helper.c @@ -38,11 +39,12 @@ noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \ glusterd-mgmt.h glusterd-messages.h glusterd-peer-utils.h \ glusterd-statedump.h glusterd-snapshot-utils.h glusterd-geo-rep.h \ glusterd-conn-mgmt.h glusterd-conn-helper.h glusterd-proc-mgmt.h \ - glusterd-svc-mgmt.h glusterd-shd-svc.h glusterd-nfs-svc.h \ + glusterd-svc-mgmt.h glusterd-nfs-svc.h \ glusterd-quotad-svc.h glusterd-svc-helper.h glusterd-snapd-svc.h \ glusterd-snapd-svc-helper.h glusterd-rcu.h glusterd-bitd-svc.h \ glusterd-scrub-svc.h glusterd-server-quorum.h glusterd-errno.h \ glusterd-tierd-svc.h glusterd-tierd-svc-helper.h \ + glusterd-shd-svc.h glusterd-shd-svc-helper.h \ glusterd-gfproxyd-svc.h glusterd-gfproxyd-svc-helper.h \ $(CONTRIBDIR)/userspace-rcu/rculist-extra.h diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c index 001eac4d39e..e08ea7be215 100644 --- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c @@ -2831,7 +2831,7 @@ glusterd_op_remove_brick(dict_t *dict, char **op_errstr) } if (start_remove && volinfo->status == GLUSTERD_STATUS_STARTED) { - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(volinfo); if (ret) { gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_RECONF_FAIL, "Unable to reconfigure NFS-Server"); diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c index c1ce2dfda37..09f0a35dc45 100644 --- a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c @@ -138,3 +138,45 @@ glusterd_conn_build_socket_filepath(char *rundir, uuid_t uuid, char *socketpath, glusterd_set_socket_filepath(sockfilepath, socketpath, len); return 0; } + +int +__glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata, + rpc_clnt_event_t event, void *data) +{ + glusterd_conf_t *conf = THIS->private; + glusterd_svc_proc_t *mux_proc = mydata; + int ret = -1; + + /* Silently ignoring this error, exactly like the current + * implementation */ + if (!mux_proc) + return 0; + + if (event == RPC_CLNT_DESTROY) { + /*RPC_CLNT_DESTROY will only called after mux_proc detached from the + * list. So it is safe to call without lock. Processing + * RPC_CLNT_DESTROY under a lock will lead to deadlock. + */ + if (mux_proc->data) { + glusterd_volinfo_unref(mux_proc->data); + mux_proc->data = NULL; + } + GF_FREE(mux_proc); + ret = 0; + } else { + pthread_mutex_lock(&conf->attach_lock); + { + ret = mux_proc->notify(mux_proc, event); + } + pthread_mutex_unlock(&conf->attach_lock); + } + return ret; +} + +int +glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata, + rpc_clnt_event_t event, void *data) +{ + return glusterd_big_locked_notify(rpc, mydata, event, data, + __glusterd_muxsvc_conn_common_notify); +} diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h index 602c0ba7b84..d1c4607795a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h +++ b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h @@ -43,9 +43,11 @@ glusterd_conn_disconnect(glusterd_conn_t *conn); int glusterd_conn_common_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, void *data); +int +glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata, + rpc_clnt_event_t event, void *data); int32_t glusterd_conn_build_socket_filepath(char *rundir, uuid_t uuid, char *socketpath, int len); - #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c index f9c8617526b..b01fd4da24b 100644 --- a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c @@ -370,6 +370,7 @@ int glusterd_gfproxydsvc_restart() { glusterd_volinfo_t *volinfo = NULL; + glusterd_volinfo_t *tmp = NULL; int ret = -1; xlator_t *this = THIS; glusterd_conf_t *conf = NULL; @@ -380,7 +381,7 @@ glusterd_gfproxydsvc_restart() conf = this->private; GF_VALIDATE_OR_GOTO(this->name, conf, out); - cds_list_for_each_entry(volinfo, &conf->volumes, vol_list) + cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list) { /* Start per volume gfproxyd svc */ if (volinfo->status == GLUSTERD_STATUS_STARTED) { diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index eb362eb9c1f..58f8b4559d3 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -5775,6 +5775,11 @@ glusterd_get_state(rpcsvc_request_t *req, dict_t *dict) GF_FREE(rebal_data); + fprintf(fp, "Volume%d.shd_svc.online_status: %s\n", count, + volinfo->shd.svc.online ? "Online" : "Offline"); + fprintf(fp, "Volume%d.shd_svc.inited: %s\n", count, + volinfo->shd.svc.inited ? "True" : "False"); + if (volinfo->type == GF_CLUSTER_TYPE_TIER) { ret = glusterd_volume_get_hot_tier_type_str(volinfo, &hot_tier_type_str); @@ -5844,12 +5849,6 @@ glusterd_get_state(rpcsvc_request_t *req, dict_t *dict) fprintf(fp, "\n[Services]\n"); - if (priv->shd_svc.inited) { - fprintf(fp, "svc%d.name: %s\n", ++count, priv->shd_svc.name); - fprintf(fp, "svc%d.online_status: %s\n\n", count, - priv->shd_svc.online ? "Online" : "Offline"); - } - if (priv->nfs_svc.inited) { fprintf(fp, "svc%d.name: %s\n", ++count, priv->nfs_svc.name); fprintf(fp, "svc%d.online_status: %s\n\n", count, diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c index 5599a639f53..1ba58c3ab9e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handshake.c +++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c @@ -30,6 +30,7 @@ #include "rpcsvc.h" #include "rpc-common-xdr.h" #include "glusterd-gfproxyd-svc-helper.h" +#include "glusterd-shd-svc-helper.h" extern struct rpc_clnt_program gd_peer_prog; extern struct rpc_clnt_program gd_mgmt_prog; @@ -328,6 +329,26 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, goto out; } + volid_ptr = strstr(volume_id, "shd/"); + if (volid_ptr) { + volid_ptr = strchr(volid_ptr, '/'); + if (!volid_ptr) { + ret = -1; + goto out; + } + volid_ptr++; + + ret = glusterd_volinfo_find(volid_ptr, &volinfo); + if (ret == -1) { + gf_log(this->name, GF_LOG_ERROR, "Couldn't find volinfo"); + goto out; + } + + glusterd_svc_build_shd_volfile_path(volinfo, path, path_len); + ret = 0; + goto out; + } + volid_ptr = strstr(volume_id, "/snaps/"); if (volid_ptr) { ret = get_snap_volname_and_volinfo(volid_ptr, &volname, &volinfo); diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h index 7a784db0372..17052cee263 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h +++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h @@ -51,6 +51,7 @@ typedef enum gf_gld_mem_types_ { gf_gld_mt_missed_snapinfo_t, gf_gld_mt_snap_create_args_t, gf_gld_mt_glusterd_brick_proc_t, + gf_gld_mt_glusterd_svc_proc_t, gf_gld_mt_end, } gf_gld_mem_types_t; #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h index 1a4bd5499e5..273ac800576 100644 --- a/xlators/mgmt/glusterd/src/glusterd-messages.h +++ b/xlators/mgmt/glusterd/src/glusterd-messages.h @@ -298,6 +298,8 @@ GLFS_MSGID( GD_MSG_LOCALTIME_LOGGING_ENABLE, GD_MSG_LOCALTIME_LOGGING_DISABLE, GD_MSG_PORTS_EXHAUSTED, GD_MSG_CHANGELOG_GET_FAIL, GD_MSG_MANAGER_FUNCTION_FAILED, - GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL); + GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL, GD_MSG_SHD_START_FAIL, + GD_MSG_SHD_OBJ_GET_FAIL, GD_MSG_SVC_ATTACH_FAIL, GD_MSG_ATTACH_INFO, + GD_MSG_DETACH_INFO, GD_MSG_SVC_DETACH_FAIL); #endif /* !_GLUSTERD_MESSAGES_H_ */ diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index 84c34f1fe4a..115622d35c6 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -44,6 +44,7 @@ #include "glusterd-snapshot-utils.h" #include "glusterd-svc-mgmt.h" #include "glusterd-svc-helper.h" +#include "glusterd-shd-svc-helper.h" #include "glusterd-shd-svc.h" #include "glusterd-nfs-svc.h" #include "glusterd-quotad-svc.h" @@ -2202,6 +2203,11 @@ glusterd_options_reset(glusterd_volinfo_t *volinfo, char *key, if (ret) goto out; + svc = &(volinfo->shd.svc); + ret = svc->reconfigure(volinfo); + if (ret) + goto out; + ret = glusterd_create_volfiles_and_notify_services(volinfo); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL, @@ -2216,7 +2222,7 @@ glusterd_options_reset(glusterd_volinfo_t *volinfo, char *key, goto out; if (GLUSTERD_STATUS_STARTED == volinfo->status) { - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(volinfo); if (ret) goto out; } @@ -2660,6 +2666,11 @@ glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict, if (ret) goto out; + svc = &(volinfo->shd.svc); + ret = svc->reconfigure(volinfo); + if (ret) + goto out; + ret = glusterd_create_volfiles_and_notify_services(volinfo); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, @@ -2673,7 +2684,7 @@ glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict, } } if (svcs_reconfigure) { - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(NULL); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL, "Unable to restart " @@ -3054,6 +3065,11 @@ glusterd_op_set_volume(dict_t *dict, char **errstr) if (ret) goto out; + svc = &(volinfo->shd.svc); + ret = svc->reconfigure(volinfo); + if (ret) + goto out; + ret = glusterd_create_volfiles_and_notify_services(volinfo); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL, @@ -3069,7 +3085,7 @@ glusterd_op_set_volume(dict_t *dict, char **errstr) goto out; if (GLUSTERD_STATUS_STARTED == volinfo->status) { - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(volinfo); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL, "Unable to restart services"); @@ -3102,6 +3118,11 @@ glusterd_op_set_volume(dict_t *dict, char **errstr) if (ret) goto out; + svc = &(volinfo->shd.svc); + ret = svc->reconfigure(volinfo); + if (ret) + goto out; + ret = glusterd_create_volfiles_and_notify_services(volinfo); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL, @@ -3117,7 +3138,7 @@ glusterd_op_set_volume(dict_t *dict, char **errstr) goto out; if (GLUSTERD_STATUS_STARTED == volinfo->status) { - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(volinfo); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL, "Unable to restart services"); @@ -3324,7 +3345,7 @@ glusterd_op_stats_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) goto out; if (GLUSTERD_STATUS_STARTED == volinfo->status) { - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(volinfo); if (ret) goto out; } @@ -3607,14 +3628,6 @@ glusterd_op_status_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) other_count++; node_count++; - } else if ((cmd & GF_CLI_STATUS_SHD) != 0) { - ret = glusterd_add_node_to_dict(priv->shd_svc.name, rsp_dict, 0, - vol_opts); - if (ret) - goto out; - other_count++; - node_count++; - } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) { ret = glusterd_add_node_to_dict(priv->quotad_svc.name, rsp_dict, 0, vol_opts); @@ -3648,6 +3661,12 @@ glusterd_op_status_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) goto out; other_count++; node_count++; + } else if ((cmd & GF_CLI_STATUS_SHD) != 0) { + ret = glusterd_add_shd_to_dict(volinfo, rsp_dict, other_index); + if (ret) + goto out; + other_count++; + node_count++; } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) { ret = dict_get_strn(dict, "brick", SLEN("brick"), &brick); if (ret) @@ -3710,6 +3729,19 @@ glusterd_op_status_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) node_count++; } + if (glusterd_is_shd_compatible_volume(volinfo)) { + shd_enabled = gd_is_self_heal_enabled(volinfo, vol_opts); + if (shd_enabled) { + ret = glusterd_add_shd_to_dict(volinfo, rsp_dict, + other_index); + if (ret) + goto out; + other_count++; + other_index++; + node_count++; + } + } + nfs_disabled = dict_get_str_boolean(vol_opts, NFS_DISABLE_MAP_KEY, _gf_false); if (!nfs_disabled) { @@ -3722,18 +3754,6 @@ glusterd_op_status_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) node_count++; } - if (glusterd_is_shd_compatible_volume(volinfo)) - shd_enabled = gd_is_self_heal_enabled(volinfo, vol_opts); - if (shd_enabled) { - ret = glusterd_add_node_to_dict(priv->shd_svc.name, rsp_dict, - other_index, vol_opts); - if (ret) - goto out; - other_count++; - node_count++; - other_index++; - } - if (glusterd_is_volume_quota_enabled(volinfo)) { ret = glusterd_add_node_to_dict(priv->quotad_svc.name, rsp_dict, other_index, vol_opts); @@ -6831,16 +6851,18 @@ glusterd_shd_select_brick_xlator(dict_t *dict, gf_xl_afr_op_t heal_op, int ret = -1; glusterd_conf_t *priv = NULL; xlator_t *this = NULL; + glusterd_svc_t *svc = NULL; this = THIS; GF_ASSERT(this); priv = this->private; GF_ASSERT(priv); + svc = &(volinfo->shd.svc); switch (heal_op) { case GF_SHD_OP_INDEX_SUMMARY: case GF_SHD_OP_STATISTICS_HEAL_COUNT: - if (!priv->shd_svc.online) { + if (!svc->online) { if (!rsp_dict) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_NULL, "Received " @@ -6861,7 +6883,7 @@ glusterd_shd_select_brick_xlator(dict_t *dict, gf_xl_afr_op_t heal_op, break; case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA: - if (!priv->shd_svc.online) { + if (!svc->online) { if (!rsp_dict) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_NULL, "Received " @@ -6996,7 +7018,7 @@ glusterd_bricks_select_heal_volume(dict_t *dict, char **op_errstr, ret = -1; goto out; } else { - pending_node->node = &(priv->shd_svc); + pending_node->node = &(volinfo->shd.svc); pending_node->type = GD_NODE_SHD; cds_list_add_tail(&pending_node->list, selected); pending_node = NULL; @@ -7130,6 +7152,7 @@ glusterd_bricks_select_status_volume(dict_t *dict, char **op_errstr, glusterd_pending_node_t *pending_node = NULL; xlator_t *this = NULL; glusterd_conf_t *priv = NULL; + glusterd_svc_t *svc = NULL; GF_ASSERT(dict); @@ -7225,7 +7248,8 @@ glusterd_bricks_select_status_volume(dict_t *dict, char **op_errstr, ret = 0; } else if ((cmd & GF_CLI_STATUS_SHD) != 0) { - if (!priv->shd_svc.online) { + svc = &(volinfo->shd.svc); + if (!svc->online) { ret = -1; gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SELF_HEALD_DISABLED, "Self-heal daemon is not running"); @@ -7237,7 +7261,7 @@ glusterd_bricks_select_status_volume(dict_t *dict, char **op_errstr, ret = -1; goto out; } - pending_node->node = &(priv->shd_svc); + pending_node->node = svc; pending_node->type = GD_NODE_SHD; pending_node->index = 0; cds_list_add_tail(&pending_node->list, selected); diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c new file mode 100644 index 00000000000..91967584f35 --- /dev/null +++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c @@ -0,0 +1,140 @@ +/* + Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "glusterd.h" +#include "glusterd-utils.h" +#include "glusterd-shd-svc-helper.h" +#include "glusterd-messages.h" +#include "glusterd-volgen.h" + +void +glusterd_svc_build_shd_socket_filepath(glusterd_volinfo_t *volinfo, char *path, + int path_len) +{ + char sockfilepath[PATH_MAX] = { + 0, + }; + char rundir[PATH_MAX] = { + 0, + }; + int32_t len = 0; + glusterd_conf_t *priv = THIS->private; + + if (!priv) + return; + + GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv); + len = snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir, + uuid_utoa(MY_UUID)); + if ((len < 0) || (len >= sizeof(sockfilepath))) { + sockfilepath[0] = 0; + } + + glusterd_set_socket_filepath(sockfilepath, path, path_len); +} + +void +glusterd_svc_build_shd_pidfile(glusterd_volinfo_t *volinfo, char *path, + int path_len) +{ + char rundir[PATH_MAX] = { + 0, + }; + glusterd_conf_t *priv = THIS->private; + + if (!priv) + return; + + GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv); + + snprintf(path, path_len, "%s/%s-shd.pid", rundir, volinfo->volname); +} + +void +glusterd_svc_build_shd_volfile_path(glusterd_volinfo_t *volinfo, char *path, + int path_len) +{ + char workdir[PATH_MAX] = { + 0, + }; + glusterd_conf_t *priv = THIS->private; + + if (!priv) + return; + + GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv); + + snprintf(path, path_len, "%s/%s-shd.vol", workdir, volinfo->volname); +} + +void +glusterd_svc_build_shd_logdir(char *logdir, char *volname, size_t len) +{ + snprintf(logdir, len, "%s/shd/%s", DEFAULT_LOG_FILE_DIRECTORY, volname); +} + +void +glusterd_svc_build_shd_logfile(char *logfile, char *logdir, size_t len) +{ + snprintf(logfile, len, "%s/shd.log", logdir); +} + +void +glusterd_shd_svcproc_cleanup(glusterd_shdsvc_t *shd) +{ + glusterd_svc_proc_t *svc_proc = NULL; + glusterd_svc_t *svc = NULL; + glusterd_conf_t *conf = NULL; + gf_boolean_t need_unref = _gf_false; + rpc_clnt_t *rpc = NULL; + + conf = THIS->private; + if (!conf) + return; + + GF_VALIDATE_OR_GOTO(THIS->name, conf, out); + GF_VALIDATE_OR_GOTO(THIS->name, shd, out); + + svc = &shd->svc; + shd->attached = _gf_false; + + if (svc->conn.rpc) { + rpc_clnt_unref(svc->conn.rpc); + svc->conn.rpc = NULL; + } + + pthread_mutex_lock(&conf->attach_lock); + { + svc_proc = svc->svc_proc; + svc->svc_proc = NULL; + svc->inited = _gf_false; + cds_list_del_init(&svc->mux_svc); + glusterd_unlink_file(svc->proc.pidfile); + + if (svc_proc && cds_list_empty(&svc_proc->svcs)) { + cds_list_del_init(&svc_proc->svc_proc_list); + /* We cannot free svc_proc list from here. Because + * if there are pending events on the rpc, it will + * try to access the corresponding svc_proc, so unrefing + * rpc request and then cleaning up the memory is carried + * from the notify function upon RPC_CLNT_DESTROY destroy. + */ + need_unref = _gf_true; + rpc = svc_proc->rpc; + svc_proc->rpc = NULL; + } + } + pthread_mutex_unlock(&conf->attach_lock); + /*rpc unref has to be performed outside the lock*/ + if (need_unref && rpc) + rpc_clnt_unref(rpc); +out: + return; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h new file mode 100644 index 00000000000..c70702c21f0 --- /dev/null +++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h @@ -0,0 +1,45 @@ +/* + Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _GLUSTERD_SHD_SVC_HELPER_H_ +#define _GLUSTERD_SHD_SVC_HELPER_H_ + +#include "glusterd.h" +#include "glusterd-svc-mgmt.h" + +void +glusterd_svc_build_shd_socket_filepath(glusterd_volinfo_t *volinfo, char *path, + int path_len); + +void +glusterd_svc_build_shd_pidfile(glusterd_volinfo_t *volinfo, char *path, + int path_len); + +void +glusterd_svc_build_shd_volfile_path(glusterd_volinfo_t *volinfo, char *path, + int path_len); + +void +glusterd_svc_build_shd_logdir(char *logdir, char *volname, size_t len); + +void +glusterd_svc_build_shd_logfile(char *logfile, char *logdir, size_t len); + +void +glusterd_shd_svcproc_cleanup(glusterd_shdsvc_t *shd); + +int +glusterd_recover_shd_attach_failure(glusterd_volinfo_t *volinfo, + glusterd_svc_t *svc, int flags); + +int +glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo); + +#endif diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c index f5379b0270b..47898434380 100644 --- a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c @@ -13,9 +13,10 @@ #include "glusterd.h" #include "glusterd-utils.h" #include "glusterd-volgen.h" -#include "glusterd-svc-mgmt.h" #include "glusterd-shd-svc.h" +#include "glusterd-shd-svc-helper.h" #include "glusterd-svc-helper.h" +#include "glusterd-store.h" #define GD_SHD_PROCESS_NAME "--process-name" char *shd_svc_name = "glustershd"; @@ -23,27 +24,145 @@ char *shd_svc_name = "glustershd"; void glusterd_shdsvc_build(glusterd_svc_t *svc) { + int ret = -1; + ret = snprintf(svc->name, sizeof(svc->name), "%s", shd_svc_name); + if (ret < 0) + return; + + CDS_INIT_LIST_HEAD(&svc->mux_svc); svc->manager = glusterd_shdsvc_manager; svc->start = glusterd_shdsvc_start; - svc->stop = glusterd_svc_stop; + svc->stop = glusterd_shdsvc_stop; + svc->reconfigure = glusterd_shdsvc_reconfigure; } int -glusterd_shdsvc_init(glusterd_svc_t *svc) +glusterd_shdsvc_init(void *data, glusterd_conn_t *mux_conn, + glusterd_svc_proc_t *mux_svc) { - return glusterd_svc_init(svc, shd_svc_name); + int ret = -1; + char rundir[PATH_MAX] = { + 0, + }; + char sockpath[PATH_MAX] = { + 0, + }; + char pidfile[PATH_MAX] = { + 0, + }; + char volfile[PATH_MAX] = { + 0, + }; + char logdir[PATH_MAX] = { + 0, + }; + char logfile[PATH_MAX] = { + 0, + }; + char volfileid[256] = {0}; + glusterd_svc_t *svc = NULL; + glusterd_volinfo_t *volinfo = NULL; + glusterd_conf_t *priv = NULL; + glusterd_muxsvc_conn_notify_t notify = NULL; + xlator_t *this = NULL; + char *volfileserver = NULL; + int32_t len = 0; + + this = THIS; + GF_VALIDATE_OR_GOTO(THIS->name, this, out); + + priv = this->private; + GF_VALIDATE_OR_GOTO(this->name, priv, out); + + volinfo = data; + GF_VALIDATE_OR_GOTO(this->name, data, out); + GF_VALIDATE_OR_GOTO(this->name, mux_svc, out); + + svc = &(volinfo->shd.svc); + + ret = snprintf(svc->name, sizeof(svc->name), "%s", shd_svc_name); + if (ret < 0) + goto out; + + notify = glusterd_muxsvc_common_rpc_notify; + glusterd_store_perform_node_state_store(volinfo); + + GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv); + glusterd_svc_create_rundir(rundir); + + glusterd_svc_build_shd_logdir(logdir, volinfo->volname, sizeof(logdir)); + glusterd_svc_build_shd_logfile(logfile, logdir, sizeof(logfile)); + + /* Initialize the connection mgmt */ + if (mux_conn && mux_svc->rpc) { + /* multiplexed svc */ + svc->conn.frame_timeout = mux_conn->frame_timeout; + /* This will be unrefed from glusterd_shd_svcproc_cleanup*/ + svc->conn.rpc = rpc_clnt_ref(mux_svc->rpc); + ret = snprintf(svc->conn.sockpath, sizeof(svc->conn.sockpath), "%s", + mux_conn->sockpath); + } else { + ret = mkdir_p(logdir, 0755, _gf_true); + if ((ret == -1) && (EEXIST != errno)) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED, + "Unable to create logdir %s", logdir); + goto out; + } + + glusterd_svc_build_shd_socket_filepath(volinfo, sockpath, + sizeof(sockpath)); + ret = glusterd_muxsvc_conn_init(&(svc->conn), mux_svc, sockpath, 600, + notify); + if (ret) + goto out; + /* This will be unrefed when the last svcs is detached from the list */ + if (!mux_svc->rpc) + mux_svc->rpc = rpc_clnt_ref(svc->conn.rpc); + } + + /* Initialize the process mgmt */ + glusterd_svc_build_shd_pidfile(volinfo, pidfile, sizeof(pidfile)); + glusterd_svc_build_shd_volfile_path(volinfo, volfile, PATH_MAX); + len = snprintf(volfileid, sizeof(volfileid), "shd/%s", volinfo->volname); + if ((len < 0) || (len >= sizeof(volfileid))) { + ret = -1; + goto out; + } + + if (dict_get_strn(this->options, "transport.socket.bind-address", + SLEN("transport.socket.bind-address"), + &volfileserver) != 0) { + volfileserver = "localhost"; + } + ret = glusterd_proc_init(&(svc->proc), shd_svc_name, pidfile, logdir, + logfile, volfile, volfileid, volfileserver); + if (ret) + goto out; + +out: + gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret); + return ret; } -static int -glusterd_shdsvc_create_volfile() +int +glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo) { char filepath[PATH_MAX] = { 0, }; + int ret = -1; - glusterd_conf_t *conf = THIS->private; dict_t *mod_dict = NULL; + glusterd_svc_build_shd_volfile_path(volinfo, filepath, PATH_MAX); + if (!glusterd_is_shd_compatible_volume(volinfo)) { + /* If volfile exist, delete it. This case happens when we + * change from replica/ec to distribute. + */ + (void)glusterd_unlink_file(filepath); + ret = 0; + goto out; + } mod_dict = dict_new(); if (!mod_dict) goto out; @@ -64,9 +183,7 @@ glusterd_shdsvc_create_volfile() if (ret) goto out; - glusterd_svc_build_volfile_path(shd_svc_name, conf->workdir, filepath, - sizeof(filepath)); - ret = glusterd_create_global_volfile(build_shd_graph, filepath, mod_dict); + ret = glusterd_shdsvc_generate_volfile(volinfo, filepath, mod_dict); if (ret) { gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL, "Failed to create volfile"); @@ -81,26 +198,89 @@ out: return ret; } +gf_boolean_t +glusterd_svcs_shd_compatible_volumes_stopped(glusterd_svc_t *svc) +{ + glusterd_svc_proc_t *svc_proc = NULL; + glusterd_shdsvc_t *shd = NULL; + glusterd_svc_t *temp_svc = NULL; + glusterd_volinfo_t *volinfo = NULL; + gf_boolean_t comp = _gf_false; + glusterd_conf_t *conf = THIS->private; + + GF_VALIDATE_OR_GOTO("glusterd", conf, out); + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + pthread_mutex_lock(&conf->attach_lock); + { + svc_proc = svc->svc_proc; + if (!svc_proc) + goto unlock; + cds_list_for_each_entry(temp_svc, &svc_proc->svcs, mux_svc) + { + /* Get volinfo->shd from svc object */ + shd = cds_list_entry(svc, glusterd_shdsvc_t, svc); + if (!shd) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL, + "Failed to get shd object " + "from shd service"); + goto unlock; + } + + /* Get volinfo from shd */ + volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd); + if (!volinfo) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, + "Failed to get volinfo from " + "from shd"); + goto unlock; + } + if (!glusterd_is_shd_compatible_volume(volinfo)) + continue; + if (volinfo->status == GLUSTERD_STATUS_STARTED) + goto unlock; + } + comp = _gf_true; + } +unlock: + pthread_mutex_unlock(&conf->attach_lock); +out: + return comp; +} + int glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags) { - int ret = 0; + int ret = -1; glusterd_volinfo_t *volinfo = NULL; - if (!svc->inited) { - ret = glusterd_shdsvc_init(svc); - if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_SHDSVC, - "Failed to init shd " - "service"); - goto out; - } else { - svc->inited = _gf_true; - gf_msg_debug(THIS->name, 0, "shd service initialized"); + volinfo = data; + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + GF_VALIDATE_OR_GOTO("glusterd", volinfo, out); + + if (volinfo) + glusterd_volinfo_ref(volinfo); + + ret = glusterd_shdsvc_create_volfile(volinfo); + if (ret) + goto out; + + if (!glusterd_is_shd_compatible_volume(volinfo)) { + ret = 0; + if (svc->inited) { + /* This means glusterd was running for this volume and now + * it was converted to a non-shd volume. So just stop the shd + */ + ret = svc->stop(svc, SIGTERM); } + goto out; } - volinfo = data; + ret = glusterd_shd_svc_mux_init(volinfo, svc); + if (ret) { + gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_SHDSVC, + "Failed to init shd service"); + goto out; + } /* If all the volumes are stopped or all shd compatible volumes * are stopped then stop the service if: @@ -110,31 +290,26 @@ glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags) * - volinfo is NULL or * - volinfo is present and volume is shd compatible */ - if (glusterd_are_all_volumes_stopped() || - glusterd_all_shd_compatible_volumes_stopped()) { - if (!(volinfo && !glusterd_is_shd_compatible_volume(volinfo))) { - ret = svc->stop(svc, SIGTERM); - } - } else { - if (!(volinfo && !glusterd_is_shd_compatible_volume(volinfo))) { - ret = glusterd_shdsvc_create_volfile(); - if (ret) - goto out; - - ret = svc->stop(svc, SIGTERM); - if (ret) - goto out; + if (glusterd_svcs_shd_compatible_volumes_stopped(svc)) { + /* TODO + * Take a lock and detach all svc's to stop the process + * also reset the init flag + */ + ret = svc->stop(svc, SIGTERM); + } else if (volinfo) { + ret = svc->stop(svc, SIGTERM); + if (ret) + goto out; + if (volinfo->status == GLUSTERD_STATUS_STARTED) { ret = svc->start(svc, flags); if (ret) goto out; - - ret = glusterd_conn_connect(&(svc->conn)); - if (ret) - goto out; } } out: + if (volinfo) + glusterd_volinfo_unref(volinfo); if (ret) gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name); gf_msg_debug(THIS->name, 0, "Returning %d", ret); @@ -143,7 +318,7 @@ out: } int -glusterd_shdsvc_start(glusterd_svc_t *svc, int flags) +glusterd_new_shd_svc_start(glusterd_svc_t *svc, int flags) { int ret = -1; char glusterd_uuid_option[PATH_MAX] = {0}; @@ -188,31 +363,136 @@ glusterd_shdsvc_start(glusterd_svc_t *svc, int flags) goto out; ret = glusterd_svc_start(svc, flags, cmdline); + if (ret) + goto out; + ret = glusterd_conn_connect(&(svc->conn)); out: if (cmdline) dict_unref(cmdline); + return ret; +} +int +glusterd_recover_shd_attach_failure(glusterd_volinfo_t *volinfo, + glusterd_svc_t *svc, int flags) +{ + int ret = -1; + glusterd_svc_proc_t *mux_proc = NULL; + glusterd_conf_t *conf = NULL; + + conf = THIS->private; + + if (!conf || !volinfo || !svc) + return -1; + glusterd_shd_svcproc_cleanup(&volinfo->shd); + mux_proc = glusterd_svcprocess_new(); + if (!mux_proc) { + return -1; + } + ret = glusterd_shdsvc_init(volinfo, NULL, mux_proc); + if (ret) + return -1; + pthread_mutex_lock(&conf->attach_lock); + { + cds_list_add_tail(&mux_proc->svc_proc_list, &conf->shd_procs); + svc->svc_proc = mux_proc; + cds_list_del_init(&svc->mux_svc); + cds_list_add_tail(&svc->mux_svc, &mux_proc->svcs); + } + pthread_mutex_unlock(&conf->attach_lock); + + ret = glusterd_new_shd_svc_start(svc, flags); + if (!ret) { + volinfo->shd.attached = _gf_true; + } + return ret; +} + +int +glusterd_shdsvc_start(glusterd_svc_t *svc, int flags) +{ + int ret = -1; + glusterd_shdsvc_t *shd = NULL; + glusterd_volinfo_t *volinfo = NULL; + glusterd_conf_t *conf = NULL; + + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + conf = THIS->private; + GF_VALIDATE_OR_GOTO("glusterd", conf, out); + + /* Get volinfo->shd from svc object */ + shd = cds_list_entry(svc, glusterd_shdsvc_t, svc); + if (!shd) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL, + "Failed to get shd object " + "from shd service"); + return -1; + } + + /* Get volinfo from shd */ + volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd); + if (!volinfo) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, + "Failed to get volinfo from " + "from shd"); + return -1; + } + + if (volinfo->status != GLUSTERD_STATUS_STARTED) + return -1; + + glusterd_volinfo_ref(volinfo); + if (!svc->inited) { + ret = glusterd_shd_svc_mux_init(volinfo, svc); + if (ret) + goto out; + } + + if (shd->attached) { + ret = glusterd_attach_svc(svc, volinfo, flags); + if (ret) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, + "Failed to attach shd svc(volume=%s) to pid=%d. Starting" + "a new process", + volinfo->volname, glusterd_proc_get_pid(&svc->proc)); + ret = glusterd_recover_shd_attach_failure(volinfo, svc, flags); + } + goto out; + } + ret = glusterd_new_shd_svc_start(svc, flags); + if (!ret) { + shd->attached = _gf_true; + } +out: + if (volinfo) + glusterd_volinfo_unref(volinfo); gf_msg_debug(THIS->name, 0, "Returning %d", ret); return ret; } int -glusterd_shdsvc_reconfigure() +glusterd_shdsvc_reconfigure(glusterd_volinfo_t *volinfo) { int ret = -1; xlator_t *this = NULL; - glusterd_conf_t *priv = NULL; gf_boolean_t identical = _gf_false; + dict_t *mod_dict = NULL; + glusterd_svc_t *svc = NULL; this = THIS; GF_VALIDATE_OR_GOTO("glusterd", this, out); - priv = this->private; - GF_VALIDATE_OR_GOTO(this->name, priv, out); + if (!volinfo) { + /* reconfigure will be called separately*/ + ret = 0; + goto out; + } - if (glusterd_all_shd_compatible_volumes_stopped()) + glusterd_volinfo_ref(volinfo); + svc = &(volinfo->shd.svc); + if (glusterd_svcs_shd_compatible_volumes_stopped(svc)) goto manager; /* @@ -220,8 +500,42 @@ glusterd_shdsvc_reconfigure() * and cksum i.e. "character-by-character". If YES, then * NOTHING has been changed, just return. */ - ret = glusterd_svc_check_volfile_identical(priv->shd_svc.name, - build_shd_graph, &identical); + + if (!glusterd_is_shd_compatible_volume(volinfo)) { + if (svc->inited) + goto manager; + + /* Nothing to do if not shd compatible */ + ret = 0; + goto out; + } + mod_dict = dict_new(); + if (!mod_dict) + goto out; + + ret = dict_set_uint32(mod_dict, "cluster.background-self-heal-count", 0); + if (ret) + goto out; + + ret = dict_set_str(mod_dict, "cluster.data-self-heal", "on"); + if (ret) + goto out; + + ret = dict_set_str(mod_dict, "cluster.metadata-self-heal", "on"); + if (ret) + goto out; + + ret = dict_set_int32(mod_dict, "graph-check", 1); + if (ret) + goto out; + + ret = dict_set_str(mod_dict, "cluster.entry-self-heal", "on"); + if (ret) + goto out; + + ret = glusterd_volume_svc_check_volfile_identical( + "glustershd", mod_dict, volinfo, glusterd_shdsvc_generate_volfile, + &identical); if (ret) goto out; @@ -236,8 +550,9 @@ glusterd_shdsvc_reconfigure() * changed, then inform the xlator to reconfigure the options. */ identical = _gf_false; /* RESET the FLAG */ - ret = glusterd_svc_check_topology_identical(priv->shd_svc.name, - build_shd_graph, &identical); + ret = glusterd_volume_svc_check_topology_identical( + "glustershd", mod_dict, volinfo, glusterd_shdsvc_generate_volfile, + &identical); if (ret) goto out; @@ -245,7 +560,7 @@ glusterd_shdsvc_reconfigure() * options to shd volfile, so that shd will be reconfigured. */ if (identical) { - ret = glusterd_shdsvc_create_volfile(); + ret = glusterd_shdsvc_create_volfile(volinfo); if (ret == 0) { /* Only if above PASSES */ ret = glusterd_fetchspec_notify(THIS); } @@ -253,12 +568,129 @@ glusterd_shdsvc_reconfigure() } manager: /* - * shd volfile's topology has been changed. shd server needs - * to be RESTARTED to ACT on the changed volfile. + * shd volfile's topology has been changed. volfile needs + * to be RECONFIGURED to ACT on the changed volfile. */ - ret = priv->shd_svc.manager(&(priv->shd_svc), NULL, PROC_START_NO_WAIT); + ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT); out: + if (volinfo) + glusterd_volinfo_unref(volinfo); + if (mod_dict) + dict_unref(mod_dict); gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret); return ret; } + +int +glusterd_shdsvc_restart() +{ + glusterd_volinfo_t *volinfo = NULL; + glusterd_volinfo_t *tmp = NULL; + int ret = -1; + xlator_t *this = THIS; + glusterd_conf_t *conf = NULL; + glusterd_svc_t *svc = NULL; + + GF_VALIDATE_OR_GOTO("glusterd", this, out); + + conf = this->private; + GF_VALIDATE_OR_GOTO(this->name, conf, out); + + pthread_mutex_lock(&conf->volume_lock); + cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list) + { + glusterd_volinfo_ref(volinfo); + pthread_mutex_unlock(&conf->volume_lock); + /* Start per volume shd svc */ + if (volinfo->status == GLUSTERD_STATUS_STARTED) { + svc = &(volinfo->shd.svc); + ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SHD_START_FAIL, + "Couldn't start shd for " + "vol: %s on restart", + volinfo->volname); + gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s", + volinfo->volname, svc->name); + glusterd_volinfo_unref(volinfo); + goto out; + } + } + glusterd_volinfo_unref(volinfo); + pthread_mutex_lock(&conf->volume_lock); + } + pthread_mutex_unlock(&conf->volume_lock); +out: + return ret; +} + +int +glusterd_shdsvc_stop(glusterd_svc_t *svc, int sig) +{ + int ret = -1; + glusterd_svc_proc_t *svc_proc = NULL; + glusterd_shdsvc_t *shd = NULL; + glusterd_volinfo_t *volinfo = NULL; + gf_boolean_t empty = _gf_false; + glusterd_conf_t *conf = NULL; + int pid = -1; + + conf = THIS->private; + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + svc_proc = svc->svc_proc; + GF_VALIDATE_OR_GOTO("glusterd", svc_proc, out); + GF_VALIDATE_OR_GOTO("glusterd", conf, out); + + /* Get volinfo->shd from svc object */ + shd = cds_list_entry(svc, glusterd_shdsvc_t, svc); + if (!shd) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL, + "Failed to get shd object " + "from shd service"); + return -1; + } + + /* Get volinfo from shd */ + volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd); + if (!volinfo) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, + "Failed to get volinfo from " + "from shd"); + return -1; + } + + glusterd_volinfo_ref(volinfo); + pthread_mutex_lock(&conf->attach_lock); + { + gf_is_service_running(svc->proc.pidfile, &pid); + cds_list_del_init(&svc->mux_svc); + empty = cds_list_empty(&svc_proc->svcs); + } + pthread_mutex_unlock(&conf->attach_lock); + if (empty) { + /* Unref will happen when destroying the connection */ + glusterd_volinfo_ref(volinfo); + svc_proc->data = volinfo; + ret = glusterd_svc_stop(svc, sig); + } + if (!empty && pid != -1) { + ret = glusterd_detach_svc(svc, volinfo, sig); + if (ret) + gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL, + "shd service is failed to detach volume %s from pid %d", + volinfo->volname, glusterd_proc_get_pid(&svc->proc)); + else + gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS, + "Shd service is detached for volume %s from pid %d", + volinfo->volname, glusterd_proc_get_pid(&svc->proc)); + } + svc->online = _gf_false; + (void)glusterd_unlink_file((char *)svc->proc.pidfile); + glusterd_shd_svcproc_cleanup(shd); + ret = 0; + glusterd_volinfo_unref(volinfo); +out: + gf_msg_debug(THIS->name, 0, "Returning %d", ret); + return ret; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc.h b/xlators/mgmt/glusterd/src/glusterd-shd-svc.h index 775a9d44a2c..55b409f4b69 100644 --- a/xlators/mgmt/glusterd/src/glusterd-shd-svc.h +++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc.h @@ -12,12 +12,20 @@ #define _GLUSTERD_SHD_SVC_H_ #include "glusterd-svc-mgmt.h" +#include "glusterd.h" + +typedef struct glusterd_shdsvc_ glusterd_shdsvc_t; +struct glusterd_shdsvc_ { + glusterd_svc_t svc; + gf_boolean_t attached; +}; void glusterd_shdsvc_build(glusterd_svc_t *svc); int -glusterd_shdsvc_init(glusterd_svc_t *svc); +glusterd_shdsvc_init(void *data, glusterd_conn_t *mux_conn, + glusterd_svc_proc_t *svc_proc); int glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags); @@ -27,4 +35,11 @@ glusterd_shdsvc_start(glusterd_svc_t *svc, int flags); int glusterd_shdsvc_reconfigure(); + +int +glusterd_shdsvc_restart(); + +int +glusterd_shdsvc_stop(glusterd_svc_t *svc, int sig); + #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c index 73a11a3eace..f0fcd956d30 100644 --- a/xlators/mgmt/glusterd/src/glusterd-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-sm.c @@ -748,6 +748,16 @@ glusterd_peer_detach_cleanup(glusterd_conf_t *priv) } } + if (glusterd_is_shd_compatible_volume(volinfo)) { + svc = &(volinfo->shd.svc); + ret = svc->stop(svc, SIGTERM); + if (ret) { + gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL, + "Failed " + "to stop shd daemon service"); + } + } + if (glusterd_is_gfproxyd_enabled(volinfo)) { svc = &(volinfo->gfproxyd.svc); ret = svc->stop(svc, SIGTERM); @@ -775,7 +785,7 @@ glusterd_peer_detach_cleanup(glusterd_conf_t *priv) } /*Reconfigure all daemon services upon peer detach*/ - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(NULL); if (ret) { gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL, "Failed to reconfigure all daemon services."); diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c index 56bab0717cd..1da4076573c 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c @@ -366,6 +366,7 @@ int glusterd_snapdsvc_restart() { glusterd_volinfo_t *volinfo = NULL; + glusterd_volinfo_t *tmp = NULL; int ret = 0; xlator_t *this = THIS; glusterd_conf_t *conf = NULL; @@ -376,7 +377,7 @@ glusterd_snapdsvc_restart() conf = this->private; GF_ASSERT(conf); - cds_list_for_each_entry(volinfo, &conf->volumes, vol_list) + cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list) { /* Start per volume snapd svc */ if (volinfo->status == GLUSTERD_STATUS_STARTED) { diff --git a/xlators/mgmt/glusterd/src/glusterd-statedump.c b/xlators/mgmt/glusterd/src/glusterd-statedump.c index f5ecde7ff35..69d4cf4aacb 100644 --- a/xlators/mgmt/glusterd/src/glusterd-statedump.c +++ b/xlators/mgmt/glusterd/src/glusterd-statedump.c @@ -202,9 +202,6 @@ glusterd_dump_priv(xlator_t *this) gf_proc_dump_build_key(key, "glusterd", "ping-timeout"); gf_proc_dump_write(key, "%d", priv->ping_timeout); - gf_proc_dump_build_key(key, "glusterd", "shd.online"); - gf_proc_dump_write(key, "%d", priv->shd_svc.online); - gf_proc_dump_build_key(key, "glusterd", "nfs.online"); gf_proc_dump_write(key, "%d", priv->nfs_svc.online); diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c index ca19a75ba82..e42703c0f41 100644 --- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c +++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c @@ -7,6 +7,7 @@ later), or the GNU General Public License, version 2 (GPLv2), in all cases as published by the Free Software Foundation. */ +#include <signal.h> #include <glusterfs/globals.h> #include <glusterfs/run.h> @@ -20,12 +21,14 @@ #include "glusterd-bitd-svc.h" #include "glusterd-tierd-svc.h" #include "glusterd-tierd-svc-helper.h" +#include "glusterd-shd-svc-helper.h" #include "glusterd-scrub-svc.h" #include "glusterd-svc-helper.h" #include <glusterfs/syscall.h> +#include "glusterd-snapshot-utils.h" int -glusterd_svcs_reconfigure() +glusterd_svcs_reconfigure(glusterd_volinfo_t *volinfo) { int ret = 0; xlator_t *this = THIS; @@ -43,9 +46,11 @@ glusterd_svcs_reconfigure() goto out; svc_name = "self-heald"; - ret = glusterd_shdsvc_reconfigure(); - if (ret) - goto out; + if (volinfo) { + ret = glusterd_shdsvc_reconfigure(volinfo); + if (ret) + goto out; + } if (conf->op_version == GD_OP_VERSION_MIN) goto out; @@ -69,7 +74,7 @@ out: } int -glusterd_svcs_stop() +glusterd_svcs_stop(glusterd_volinfo_t *volinfo) { int ret = 0; xlator_t *this = NULL; @@ -85,14 +90,16 @@ glusterd_svcs_stop() if (ret) goto out; - ret = glusterd_svc_stop(&(priv->shd_svc), SIGTERM); - if (ret) - goto out; - ret = glusterd_svc_stop(&(priv->quotad_svc), SIGTERM); if (ret) goto out; + if (volinfo) { + ret = glusterd_svc_stop(&(volinfo->shd.svc), PROC_START_NO_WAIT); + if (ret) + goto out; + } + ret = glusterd_svc_stop(&(priv->bitd_svc), SIGTERM); if (ret) goto out; @@ -121,12 +128,6 @@ glusterd_svcs_manager(glusterd_volinfo_t *volinfo) if (ret) goto out; - ret = conf->shd_svc.manager(&(conf->shd_svc), volinfo, PROC_START_NO_WAIT); - if (ret == -EINVAL) - ret = 0; - if (ret) - goto out; - if (conf->op_version == GD_OP_VERSION_MIN) goto out; @@ -143,6 +144,15 @@ glusterd_svcs_manager(glusterd_volinfo_t *volinfo) if (ret) goto out; + if (volinfo) { + ret = volinfo->shd.svc.manager(&(volinfo->shd.svc), volinfo, + PROC_START_NO_WAIT); + if (ret == -EINVAL) + ret = 0; + if (ret) + goto out; + } + ret = conf->scrub_svc.manager(&(conf->scrub_svc), NULL, PROC_START_NO_WAIT); if (ret == -EINVAL) ret = 0; @@ -269,3 +279,678 @@ out: GF_FREE(tmpvol); return ret; } + +int +glusterd_volume_svc_check_volfile_identical( + char *svc_name, dict_t *mode_dict, glusterd_volinfo_t *volinfo, + glusterd_vol_graph_builder_t builder, gf_boolean_t *identical) +{ + char orgvol[PATH_MAX] = { + 0, + }; + char *tmpvol = NULL; + xlator_t *this = NULL; + int ret = -1; + int need_unlink = 0; + int tmp_fd = -1; + + this = THIS; + + GF_VALIDATE_OR_GOTO(this->name, this, out); + GF_VALIDATE_OR_GOTO(this->name, identical, out); + + /* This builds volfile for volume level dameons */ + glusterd_volume_svc_build_volfile_path(svc_name, volinfo, orgvol, + sizeof(orgvol)); + + ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name); + if (ret < 0) { + goto out; + } + + /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */ + tmp_fd = mkstemp(tmpvol); + if (tmp_fd < 0) { + gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED, + "Unable to create temp file" + " %s:(%s)", + tmpvol, strerror(errno)); + ret = -1; + goto out; + } + + need_unlink = 1; + + ret = builder(volinfo, tmpvol, mode_dict); + if (ret) + goto out; + + ret = glusterd_check_files_identical(orgvol, tmpvol, identical); +out: + if (need_unlink) + sys_unlink(tmpvol); + + if (tmpvol != NULL) + GF_FREE(tmpvol); + + if (tmp_fd >= 0) + sys_close(tmp_fd); + + return ret; +} + +int +glusterd_volume_svc_check_topology_identical( + char *svc_name, dict_t *mode_dict, glusterd_volinfo_t *volinfo, + glusterd_vol_graph_builder_t builder, gf_boolean_t *identical) +{ + char orgvol[PATH_MAX] = { + 0, + }; + char *tmpvol = NULL; + glusterd_conf_t *conf = NULL; + xlator_t *this = THIS; + int ret = -1; + int tmpclean = 0; + int tmpfd = -1; + + if ((!identical) || (!this) || (!this->private)) + goto out; + + conf = this->private; + GF_VALIDATE_OR_GOTO(this->name, conf, out); + + /* This builds volfile for volume level dameons */ + glusterd_volume_svc_build_volfile_path(svc_name, volinfo, orgvol, + sizeof(orgvol)); + /* Create the temporary volfile */ + ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name); + if (ret < 0) { + goto out; + } + + /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */ + tmpfd = mkstemp(tmpvol); + if (tmpfd < 0) { + gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED, + "Unable to create temp file" + " %s:(%s)", + tmpvol, strerror(errno)); + ret = -1; + goto out; + } + + tmpclean = 1; /* SET the flag to unlink() tmpfile */ + + ret = builder(volinfo, tmpvol, mode_dict); + if (ret) + goto out; + + /* Compare the topology of volfiles */ + ret = glusterd_check_topology_identical(orgvol, tmpvol, identical); +out: + if (tmpfd >= 0) + sys_close(tmpfd); + if (tmpclean) + sys_unlink(tmpvol); + if (tmpvol != NULL) + GF_FREE(tmpvol); + return ret; +} + +void * +__gf_find_compatible_svc(gd_node_type daemon) +{ + glusterd_svc_proc_t *svc_proc = NULL; + glusterd_svc_proc_t *return_proc = NULL; + glusterd_svc_t *parent_svc = NULL; + struct cds_list_head *svc_procs = NULL; + glusterd_conf_t *conf = NULL; + int pid = -1; + + conf = THIS->private; + GF_VALIDATE_OR_GOTO("glusterd", conf, out); + + if (daemon == GD_NODE_SHD) { + svc_procs = &conf->shd_procs; + if (!svc_procs) + goto out; + } + + cds_list_for_each_entry(svc_proc, svc_procs, svc_proc_list) + { + parent_svc = cds_list_entry(svc_proc->svcs.next, glusterd_svc_t, + mux_svc); + if (!return_proc) + return_proc = svc_proc; + + /* If there is an already running shd daemons, select it. Otehrwise + * select the first one. + */ + if (parent_svc && gf_is_service_running(parent_svc->proc.pidfile, &pid)) + return (void *)svc_proc; + /* + * Logic to select one process goes here. Currently there is only one + * shd_proc. So selecting the first one; + */ + } +out: + return return_proc; +} + +glusterd_svc_proc_t * +glusterd_svcprocess_new() +{ + glusterd_svc_proc_t *new_svcprocess = NULL; + + new_svcprocess = GF_CALLOC(1, sizeof(*new_svcprocess), + gf_gld_mt_glusterd_svc_proc_t); + + if (!new_svcprocess) + return NULL; + + CDS_INIT_LIST_HEAD(&new_svcprocess->svc_proc_list); + CDS_INIT_LIST_HEAD(&new_svcprocess->svcs); + new_svcprocess->notify = glusterd_muxsvc_common_rpc_notify; + return new_svcprocess; +} + +int +glusterd_shd_svc_mux_init(glusterd_volinfo_t *volinfo, glusterd_svc_t *svc) +{ + int ret = -1; + glusterd_svc_proc_t *mux_proc = NULL; + glusterd_conn_t *mux_conn = NULL; + glusterd_conf_t *conf = NULL; + glusterd_svc_t *parent_svc = NULL; + int pid = -1; + + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + GF_VALIDATE_OR_GOTO("glusterd", volinfo, out); + conf = THIS->private; + GF_VALIDATE_OR_GOTO("glusterd", conf, out); + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + + pthread_mutex_lock(&conf->attach_lock); + { + if (!svc->inited) { + if (gf_is_service_running(svc->proc.pidfile, &pid)) { + /* Just connect is required, but we don't know what happens + * during the disconnect. So better to reattach. + */ + mux_proc = __gf_find_compatible_svc_from_pid(GD_NODE_SHD, pid); + } + + if (!mux_proc) { + if (pid != -1 && sys_access(svc->proc.pidfile, R_OK) == 0) { + /* stale pid file, unlink it. */ + kill(pid, SIGTERM); + sys_unlink(svc->proc.pidfile); + } + mux_proc = __gf_find_compatible_svc(GD_NODE_SHD); + } + if (mux_proc) { + /* Take first entry from the process */ + parent_svc = cds_list_entry(mux_proc->svcs.next, glusterd_svc_t, + mux_svc); + sys_link(parent_svc->proc.pidfile, svc->proc.pidfile); + mux_conn = &parent_svc->conn; + if (volinfo) + volinfo->shd.attached = _gf_true; + } else { + mux_proc = glusterd_svcprocess_new(); + if (!mux_proc) { + ret = -1; + goto unlock; + } + cds_list_add_tail(&mux_proc->svc_proc_list, &conf->shd_procs); + } + svc->svc_proc = mux_proc; + cds_list_del_init(&svc->mux_svc); + cds_list_add_tail(&svc->mux_svc, &mux_proc->svcs); + ret = glusterd_shdsvc_init(volinfo, mux_conn, mux_proc); + if (ret) { + pthread_mutex_unlock(&conf->attach_lock); + gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_SHDSVC, + "Failed to init shd " + "service"); + goto out; + } + gf_msg_debug(THIS->name, 0, "shd service initialized"); + svc->inited = _gf_true; + } + ret = 0; + } +unlock: + pthread_mutex_unlock(&conf->attach_lock); +out: + return ret; +} + +void * +__gf_find_compatible_svc_from_pid(gd_node_type daemon, pid_t pid) +{ + glusterd_svc_proc_t *svc_proc = NULL; + struct cds_list_head *svc_procs = NULL; + glusterd_svc_t *svc = NULL; + pid_t mux_pid = -1; + glusterd_conf_t *conf = NULL; + + conf = THIS->private; + if (!conf) + return NULL; + + if (daemon == GD_NODE_SHD) { + svc_procs = &conf->shd_procs; + if (!svc_proc) + return NULL; + } /* Can be moved to switch when mux is implemented for other daemon; */ + + cds_list_for_each_entry(svc_proc, svc_procs, svc_proc_list) + { + cds_list_for_each_entry(svc, &svc_proc->svcs, mux_svc) + { + if (gf_is_service_running(svc->proc.pidfile, &mux_pid)) { + if (mux_pid == pid) { + /*TODO + * inefficient loop, but at the moment, there is only + * one shd. + */ + return svc_proc; + } + } + } + } + return NULL; +} + +static int32_t +my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame) +{ + call_frame_t *frame = v_frame; + xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; + + GF_VALIDATE_OR_GOTO("glusterd", frame, out); + this = frame->this; + GF_VALIDATE_OR_GOTO("glusterd", this, out); + conf = this->private; + GF_VALIDATE_OR_GOTO(this->name, conf, out); + + GF_ATOMIC_DEC(conf->blockers); + + STACK_DESTROY(frame->root); +out: + return 0; +} + +static int32_t +glusterd_svc_attach_cbk(struct rpc_req *req, struct iovec *iov, int count, + void *v_frame) +{ + call_frame_t *frame = v_frame; + glusterd_volinfo_t *volinfo = NULL; + glusterd_shdsvc_t *shd = NULL; + glusterd_svc_t *svc = frame->cookie; + glusterd_svc_t *parent_svc = NULL; + glusterd_svc_proc_t *mux_proc = NULL; + glusterd_conf_t *conf = NULL; + int *flag = (int *)frame->local; + xlator_t *this = THIS; + int pid = -1; + int ret = -1; + gf_getspec_rsp rsp = { + 0, + }; + + GF_VALIDATE_OR_GOTO("glusterd", this, out); + conf = this->private; + GF_VALIDATE_OR_GOTO("glusterd", conf, out); + GF_VALIDATE_OR_GOTO("glusterd", frame, out); + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + + frame->local = NULL; + frame->cookie = NULL; + + if (!strcmp(svc->name, "glustershd")) { + /* Get volinfo->shd from svc object */ + shd = cds_list_entry(svc, glusterd_shdsvc_t, svc); + if (!shd) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL, + "Failed to get shd object " + "from shd service"); + goto out; + } + + /* Get volinfo from shd */ + volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd); + if (!volinfo) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, + "Failed to get volinfo from " + "from shd"); + goto out; + } + } + + if (!iov) { + gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, + "iov is NULL"); + ret = -1; + goto out; + } + + ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp); + if (ret < 0) { + gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, + "XDR decoding error"); + ret = -1; + goto out; + } + + if (rsp.op_ret == 0) { + pthread_mutex_lock(&conf->attach_lock); + { + if (!strcmp(svc->name, "glustershd")) { + mux_proc = svc->svc_proc; + if (mux_proc && + !gf_is_service_running(svc->proc.pidfile, &pid)) { + /* + * When svc's are restarting, there is a chance that the + * attached svc might not have updated it's pid. Because + * it was at connection stage. So in that case, we need + * to retry the pid file copy. + */ + parent_svc = cds_list_entry(mux_proc->svcs.next, + glusterd_svc_t, mux_svc); + if (parent_svc) + sys_link(parent_svc->proc.pidfile, svc->proc.pidfile); + } + } + svc->online = _gf_true; + } + pthread_mutex_unlock(&conf->attach_lock); + gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SVC_ATTACH_FAIL, + "svc %s of volume %s attached successfully to pid %d", svc->name, + volinfo->volname, glusterd_proc_get_pid(&svc->proc)); + } else { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL, + "svc %s of volume %s failed to " + "attach to pid %d. Starting a new process", + svc->name, volinfo->volname, glusterd_proc_get_pid(&svc->proc)); + if (!strcmp(svc->name, "glustershd")) { + glusterd_recover_shd_attach_failure(volinfo, svc, *flag); + } + } +out: + if (flag) { + GF_FREE(flag); + } + GF_ATOMIC_DEC(conf->blockers); + STACK_DESTROY(frame->root); + return 0; +} + +extern size_t +build_volfile_path(char *volume_id, char *path, size_t path_len, + char *trusted_str); + +int +__glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flags, + struct rpc_clnt *rpc, char *volfile_id, + int op) +{ + int ret = -1; + struct iobuf *iobuf = NULL; + struct iobref *iobref = NULL; + struct iovec iov = { + 0, + }; + char path[PATH_MAX] = { + '\0', + }; + struct stat stbuf = { + 0, + }; + int32_t spec_fd = -1; + size_t file_len = -1; + char *volfile_content = NULL; + ssize_t req_size = 0; + call_frame_t *frame = NULL; + gd1_mgmt_brick_op_req brick_req; + void *req = &brick_req; + void *errlbl = &&err; + struct rpc_clnt_connection *conn; + xlator_t *this = THIS; + glusterd_conf_t *conf = THIS->private; + extern struct rpc_clnt_program gd_brick_prog; + fop_cbk_fn_t cbkfn = my_callback; + + if (!rpc) { + gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PARAM_NULL, + "called with null rpc"); + return -1; + } + + conn = &rpc->conn; + if (!conn->connected || conn->disconnected) { + gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONNECT_RETURNED, + "not connected yet"); + return -1; + } + + brick_req.op = op; + brick_req.name = volfile_id; + brick_req.input.input_val = NULL; + brick_req.input.input_len = 0; + + frame = create_frame(this, this->ctx->pool); + if (!frame) { + goto *errlbl; + } + + if (op == GLUSTERD_SVC_ATTACH) { + (void)build_volfile_path(volfile_id, path, sizeof(path), NULL); + + ret = sys_stat(path, &stbuf); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL, + "Unable to stat %s (%s)", path, strerror(errno)); + ret = -EINVAL; + goto *errlbl; + } + + file_len = stbuf.st_size; + volfile_content = GF_MALLOC(file_len + 1, gf_common_mt_char); + if (!volfile_content) { + ret = -ENOMEM; + goto *errlbl; + } + spec_fd = open(path, O_RDONLY); + if (spec_fd < 0) { + gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_SVC_ATTACH_FAIL, + "failed to read volfile %s", path); + ret = -EIO; + goto *errlbl; + } + ret = sys_read(spec_fd, volfile_content, file_len); + if (ret == file_len) { + brick_req.input.input_val = volfile_content; + brick_req.input.input_len = file_len; + } else { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL, + "read failed on path %s. File size=%" GF_PRI_SIZET + "read size=%d", + path, file_len, ret); + ret = -EIO; + goto *errlbl; + } + + frame->cookie = svc; + frame->local = GF_CALLOC(1, sizeof(int), gf_gld_mt_int); + *((int *)frame->local) = flags; + cbkfn = glusterd_svc_attach_cbk; + } + + req_size = xdr_sizeof((xdrproc_t)xdr_gd1_mgmt_brick_op_req, req); + iobuf = iobuf_get2(rpc->ctx->iobuf_pool, req_size); + if (!iobuf) { + goto *errlbl; + } + errlbl = &&maybe_free_iobuf; + + iov.iov_base = iobuf->ptr; + iov.iov_len = iobuf_pagesize(iobuf); + + iobref = iobref_new(); + if (!iobref) { + goto *errlbl; + } + errlbl = &&free_iobref; + + iobref_add(iobref, iobuf); + /* + * Drop our reference to the iobuf. The iobref should already have + * one after iobref_add, so when we unref that we'll free the iobuf as + * well. This allows us to pass just the iobref as frame->local. + */ + iobuf_unref(iobuf); + /* Set the pointer to null so we don't free it on a later error. */ + iobuf = NULL; + + /* Create the xdr payload */ + ret = xdr_serialize_generic(iov, req, (xdrproc_t)xdr_gd1_mgmt_brick_op_req); + if (ret == -1) { + goto *errlbl; + } + iov.iov_len = ret; + + /* Send the msg */ + GF_ATOMIC_INC(conf->blockers); + ret = rpc_clnt_submit(rpc, &gd_brick_prog, op, cbkfn, &iov, 1, NULL, 0, + iobref, frame, NULL, 0, NULL, 0, NULL); + GF_FREE(volfile_content); + if (spec_fd >= 0) + sys_close(spec_fd); + return ret; + +free_iobref: + iobref_unref(iobref); +maybe_free_iobuf: + if (iobuf) { + iobuf_unref(iobuf); + } +err: + GF_FREE(volfile_content); + if (spec_fd >= 0) + sys_close(spec_fd); + if (frame) + STACK_DESTROY(frame->root); + return -1; +} + +int +glusterd_attach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int flags) +{ + glusterd_conf_t *conf = THIS->private; + int ret = -1; + int tries; + rpc_clnt_t *rpc = NULL; + + GF_VALIDATE_OR_GOTO("glusterd", conf, out); + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + GF_VALIDATE_OR_GOTO("glusterd", volinfo, out); + + gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_ATTACH_INFO, + "adding svc %s (volume=%s) to existing " + "process with pid %d", + svc->name, volinfo->volname, glusterd_proc_get_pid(&svc->proc)); + + rpc = rpc_clnt_ref(svc->conn.rpc); + for (tries = 15; tries > 0; --tries) { + if (rpc) { + pthread_mutex_lock(&conf->attach_lock); + { + ret = __glusterd_send_svc_configure_req( + svc, flags, rpc, svc->proc.volfileid, GLUSTERD_SVC_ATTACH); + } + pthread_mutex_unlock(&conf->attach_lock); + if (!ret) { + volinfo->shd.attached = _gf_true; + goto out; + } + } + /* + * It might not actually be safe to manipulate the lock + * like this, but if we don't then the connection can + * never actually complete and retries are useless. + * Unfortunately, all of the alternatives (e.g. doing + * all of this in a separate thread) are much more + * complicated and risky. + * TBD: see if there's a better way + */ + synclock_unlock(&conf->big_lock); + sleep(1); + synclock_lock(&conf->big_lock); + } + ret = -1; + gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SVC_ATTACH_FAIL, + "attach failed for %s(volume=%s)", svc->name, volinfo->volname); +out: + if (rpc) + rpc_clnt_unref(rpc); + return ret; +} + +int +glusterd_detach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int sig) +{ + glusterd_conf_t *conf = THIS->private; + int ret = -1; + int tries; + rpc_clnt_t *rpc = NULL; + + GF_VALIDATE_OR_GOTO(THIS->name, conf, out); + GF_VALIDATE_OR_GOTO(THIS->name, svc, out); + GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out); + + gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DETACH_INFO, + "removing svc %s (volume=%s) from existing " + "process with pid %d", + svc->name, volinfo->volname, glusterd_proc_get_pid(&svc->proc)); + + rpc = rpc_clnt_ref(svc->conn.rpc); + for (tries = 15; tries > 0; --tries) { + if (rpc) { + /*For detach there is no flags, and we are not using sig.*/ + pthread_mutex_lock(&conf->attach_lock); + { + ret = __glusterd_send_svc_configure_req(svc, 0, svc->conn.rpc, + svc->proc.volfileid, + GLUSTERD_SVC_DETACH); + } + pthread_mutex_unlock(&conf->attach_lock); + if (!ret) { + goto out; + } + } + /* + * It might not actually be safe to manipulate the lock + * like this, but if we don't then the connection can + * never actually complete and retries are useless. + * Unfortunately, all of the alternatives (e.g. doing + * all of this in a separate thread) are much more + * complicated and risky. + * TBD: see if there's a better way + */ + synclock_unlock(&conf->big_lock); + sleep(1); + synclock_lock(&conf->big_lock); + } + ret = -1; + gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SVC_DETACH_FAIL, + "detach failed for %s(volume=%s)", svc->name, volinfo->volname); +out: + if (rpc) + rpc_clnt_unref(rpc); + return ret; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-svc-helper.h index cc98e788bbe..5def2468785 100644 --- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.h +++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.h @@ -16,10 +16,10 @@ #include "glusterd-volgen.h" int -glusterd_svcs_reconfigure(); +glusterd_svcs_reconfigure(glusterd_volinfo_t *volinfo); int -glusterd_svcs_stop(); +glusterd_svcs_stop(glusterd_volinfo_t *vol); int glusterd_svcs_manager(glusterd_volinfo_t *volinfo); @@ -41,5 +41,41 @@ int glusterd_svc_check_tier_topology_identical(char *svc_name, glusterd_volinfo_t *volinfo, gf_boolean_t *identical); +int +glusterd_volume_svc_check_volfile_identical(char *svc_name, dict_t *mode_dict, + glusterd_volinfo_t *volinfo, + glusterd_vol_graph_builder_t, + gf_boolean_t *identical); +int +glusterd_volume_svc_check_topology_identical(char *svc_name, dict_t *mode_dict, + glusterd_volinfo_t *volinfo, + glusterd_vol_graph_builder_t, + gf_boolean_t *identical); +void +glusterd_volume_svc_build_volfile_path(char *server, glusterd_volinfo_t *vol, + char *volfile, size_t len); +void * +__gf_find_compatible_svc(gd_node_type daemon); + +glusterd_svc_proc_t * +glusterd_svcprocess_new(); + +int +glusterd_shd_svc_mux_init(glusterd_volinfo_t *volinfo, glusterd_svc_t *svc); + +void * +__gf_find_compatible_svc_from_pid(gd_node_type daemon, pid_t pid); + +int +glusterd_attach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, + int flags); + +int +glusterd_detach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int sig); + +int +__glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flag, + struct rpc_clnt *rpc, char *volfile_id, + int op); #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c index 6325f60f94a..63c6b609bbd 100644 --- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c @@ -18,6 +18,7 @@ #include "glusterd-conn-mgmt.h" #include "glusterd-messages.h" #include <glusterfs/syscall.h> +#include "glusterd-shd-svc-helper.h" int glusterd_svc_create_rundir(char *rundir) @@ -167,72 +168,79 @@ glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline) GF_ASSERT(this); priv = this->private; - GF_ASSERT(priv); + GF_VALIDATE_OR_GOTO("glusterd", priv, out); + GF_VALIDATE_OR_GOTO("glusterd", svc, out); + + pthread_mutex_lock(&priv->attach_lock); + { + if (glusterd_proc_is_running(&(svc->proc))) { + ret = 0; + goto unlock; + } - if (glusterd_proc_is_running(&(svc->proc))) { - ret = 0; - goto out; - } + ret = sys_access(svc->proc.volfile, F_OK); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_NOT_FOUND, + "Volfile %s is not present", svc->proc.volfile); + goto unlock; + } - ret = sys_access(svc->proc.volfile, F_OK); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_NOT_FOUND, - "Volfile %s is not present", svc->proc.volfile); - goto out; - } + runinit(&runner); - runinit(&runner); + if (this->ctx->cmd_args.valgrind) { + len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s.log", + svc->proc.logfile, svc->name); + if ((len < 0) || (len >= PATH_MAX)) { + ret = -1; + goto unlock; + } - if (this->ctx->cmd_args.valgrind) { - len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s.log", - svc->proc.logfile, svc->name); - if ((len < 0) || (len >= PATH_MAX)) { - ret = -1; - goto out; + runner_add_args(&runner, "valgrind", "--leak-check=full", + "--trace-children=yes", "--track-origins=yes", + NULL); + runner_argprintf(&runner, "--log-file=%s", valgrind_logfile); } - runner_add_args(&runner, "valgrind", "--leak-check=full", - "--trace-children=yes", "--track-origins=yes", NULL); - runner_argprintf(&runner, "--log-file=%s", valgrind_logfile); - } - - runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s", - svc->proc.volfileserver, "--volfile-id", - svc->proc.volfileid, "-p", svc->proc.pidfile, "-l", - svc->proc.logfile, "-S", svc->conn.sockpath, NULL); + runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s", + svc->proc.volfileserver, "--volfile-id", + svc->proc.volfileid, "-p", svc->proc.pidfile, "-l", + svc->proc.logfile, "-S", svc->conn.sockpath, NULL); - if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY, - SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY), - &localtime_logging) == 0) { - if (strcmp(localtime_logging, "enable") == 0) - runner_add_arg(&runner, "--localtime-logging"); - } - if (dict_get_strn(priv->opts, GLUSTERD_DAEMON_LOG_LEVEL_KEY, - SLEN(GLUSTERD_DAEMON_LOG_LEVEL_KEY), &log_level) == 0) { - snprintf(daemon_log_level, 30, "--log-level=%s", log_level); - runner_add_arg(&runner, daemon_log_level); - } + if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY, + SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY), + &localtime_logging) == 0) { + if (strcmp(localtime_logging, "enable") == 0) + runner_add_arg(&runner, "--localtime-logging"); + } + if (dict_get_strn(priv->opts, GLUSTERD_DAEMON_LOG_LEVEL_KEY, + SLEN(GLUSTERD_DAEMON_LOG_LEVEL_KEY), + &log_level) == 0) { + snprintf(daemon_log_level, 30, "--log-level=%s", log_level); + runner_add_arg(&runner, daemon_log_level); + } - if (this->ctx->cmd_args.global_threading) { - runner_add_arg(&runner, "--global-threading"); - } + if (this->ctx->cmd_args.global_threading) { + runner_add_arg(&runner, "--global-threading"); + } - if (cmdline) - dict_foreach(cmdline, svc_add_args, (void *)&runner); + if (cmdline) + dict_foreach(cmdline, svc_add_args, (void *)&runner); - gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SVC_START_SUCCESS, - "Starting %s service", svc->name); + gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SVC_START_SUCCESS, + "Starting %s service", svc->name); - if (flags == PROC_START_NO_WAIT) { - ret = runner_run_nowait(&runner); - } else { - synclock_unlock(&priv->big_lock); - { - ret = runner_run(&runner); + if (flags == PROC_START_NO_WAIT) { + ret = runner_run_nowait(&runner); + } else { + synclock_unlock(&priv->big_lock); + { + ret = runner_run(&runner); + } + synclock_lock(&priv->big_lock); } - synclock_lock(&priv->big_lock); } - +unlock: + pthread_mutex_unlock(&priv->attach_lock); out: gf_msg_debug(this->name, 0, "Returning %d", ret); @@ -285,7 +293,8 @@ glusterd_svc_build_volfile_path(char *server, char *workdir, char *volfile, glusterd_svc_build_svcdir(server, workdir, dir, sizeof(dir)); - if (!strcmp(server, "quotad")) /*quotad has different volfile name*/ + if (!strcmp(server, "quotad")) + /*quotad has different volfile name*/ snprintf(volfile, len, "%s/%s.vol", dir, server); else snprintf(volfile, len, "%s/%s-server.vol", dir, server); @@ -370,3 +379,138 @@ glusterd_svc_common_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event) return ret; } + +void +glusterd_volume_svc_build_volfile_path(char *server, glusterd_volinfo_t *vol, + char *volfile, size_t len) +{ + GF_ASSERT(len == PATH_MAX); + + if (!strcmp(server, "glustershd")) { + glusterd_svc_build_shd_volfile_path(vol, volfile, len); + } +} + +int +glusterd_muxsvc_common_rpc_notify(glusterd_svc_proc_t *mux_proc, + rpc_clnt_event_t event) +{ + int ret = 0; + glusterd_svc_t *svc = NULL; + glusterd_svc_t *tmp = NULL; + xlator_t *this = NULL; + gf_boolean_t need_logging = _gf_false; + + this = THIS; + GF_ASSERT(this); + + if (!mux_proc) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL, + "Failed to get the svc proc data"); + return -1; + } + + /* Currently this function was used for shd svc, if this function is + * using for another svc, change ths glustershd reference. We can get + * the svc name from any of the attached svc's + */ + switch (event) { + case RPC_CLNT_CONNECT: + gf_msg_debug(this->name, 0, + "glustershd has connected with glusterd."); + gf_event(EVENT_SVC_CONNECTED, "svc_name=glustershd"); + cds_list_for_each_entry_safe(svc, tmp, &mux_proc->svcs, mux_svc) + { + if (svc->online) + continue; + svc->online = _gf_true; + } + break; + + case RPC_CLNT_DISCONNECT: + cds_list_for_each_entry_safe(svc, tmp, &mux_proc->svcs, mux_svc) + { + if (svc->online) { + if (!need_logging) + need_logging = _gf_true; + svc->online = _gf_false; + } + } + if (need_logging) { + gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NODE_DISCONNECTED, + "glustershd has disconnected from glusterd."); + gf_event(EVENT_SVC_DISCONNECTED, "svc_name=glustershd"); + } + break; + + default: + gf_msg_trace(this->name, 0, "got some other RPC event %d", event); + break; + } + + return ret; +} + +int +glusterd_muxsvc_conn_init(glusterd_conn_t *conn, glusterd_svc_proc_t *mux_proc, + char *sockpath, int frame_timeout, + glusterd_muxsvc_conn_notify_t notify) +{ + int ret = -1; + dict_t *options = NULL; + struct rpc_clnt *rpc = NULL; + xlator_t *this = THIS; + glusterd_svc_t *svc = NULL; + + options = dict_new(); + if (!this || !options) + goto out; + + svc = cds_list_entry(conn, glusterd_svc_t, conn); + if (!svc) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL, + "Failed to get the service"); + goto out; + } + + ret = rpc_transport_unix_options_build(options, sockpath, frame_timeout); + if (ret) + goto out; + + ret = dict_set_int32n(options, "transport.socket.ignore-enoent", + SLEN("transport.socket.ignore-enoent"), 1); + if (ret) + goto out; + + /* @options is free'd by rpc_transport when destroyed */ + rpc = rpc_clnt_new(options, this, (char *)svc->name, 16); + if (!rpc) { + ret = -1; + goto out; + } + + ret = rpc_clnt_register_notify(rpc, glusterd_muxsvc_conn_common_notify, + mux_proc); + if (ret) + goto out; + + ret = snprintf(conn->sockpath, sizeof(conn->sockpath), "%s", sockpath); + if (ret < 0) + goto out; + else + ret = 0; + + conn->frame_timeout = frame_timeout; + conn->rpc = rpc; + mux_proc->notify = notify; +out: + if (options) + dict_unref(options); + if (ret) { + if (rpc) { + rpc_clnt_unref(rpc); + rpc = NULL; + } + } + return ret; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h index c850bfda68f..fbc522549e0 100644 --- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h +++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h @@ -13,9 +13,12 @@ #include "glusterd-proc-mgmt.h" #include "glusterd-conn-mgmt.h" +#include "glusterd-rcu.h" struct glusterd_svc_; + typedef struct glusterd_svc_ glusterd_svc_t; +typedef struct glusterd_svc_proc_ glusterd_svc_proc_t; typedef void (*glusterd_svc_build_t)(glusterd_svc_t *svc); @@ -25,6 +28,17 @@ typedef int (*glusterd_svc_start_t)(glusterd_svc_t *svc, int flags); typedef int (*glusterd_svc_stop_t)(glusterd_svc_t *svc, int sig); typedef int (*glusterd_svc_reconfigure_t)(void *data); +typedef int (*glusterd_muxsvc_conn_notify_t)(glusterd_svc_proc_t *mux_proc, + rpc_clnt_event_t event); + +struct glusterd_svc_proc_ { + struct cds_list_head svc_proc_list; + struct cds_list_head svcs; + glusterd_muxsvc_conn_notify_t notify; + rpc_clnt_t *rpc; + void *data; +}; + struct glusterd_svc_ { char name[NAME_MAX]; glusterd_conn_t conn; @@ -35,6 +49,8 @@ struct glusterd_svc_ { gf_boolean_t online; gf_boolean_t inited; glusterd_svc_reconfigure_t reconfigure; + glusterd_svc_proc_t *svc_proc; + struct cds_list_head mux_svc; }; int @@ -69,4 +85,15 @@ glusterd_svc_reconfigure(int (*create_volfile)()); int glusterd_svc_common_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event); +int +glusterd_muxsvc_common_rpc_notify(glusterd_svc_proc_t *conn, + rpc_clnt_event_t event); + +int +glusterd_proc_get_pid(glusterd_proc_t *proc); + +int +glusterd_muxsvc_conn_init(glusterd_conn_t *conn, glusterd_svc_proc_t *mux_proc, + char *sockpath, int frame_timeout, + glusterd_muxsvc_conn_notify_t notify); #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-tier.c b/xlators/mgmt/glusterd/src/glusterd-tier.c index dd86cf504ff..e4b79dfc862 100644 --- a/xlators/mgmt/glusterd/src/glusterd-tier.c +++ b/xlators/mgmt/glusterd/src/glusterd-tier.c @@ -27,6 +27,7 @@ #include "glusterd-messages.h" #include "glusterd-mgmt.h" #include "glusterd-syncop.h" +#include "glusterd-shd-svc-helper.h" #include <sys/wait.h> #include <dlfcn.h> @@ -615,7 +616,7 @@ glusterd_op_remove_tier_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) if (cmd == GF_DEFRAG_CMD_DETACH_START && volinfo->status == GLUSTERD_STATUS_STARTED) { - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(volinfo); if (ret) { gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_RECONF_FAIL, "Unable to reconfigure NFS-Server"); diff --git a/xlators/mgmt/glusterd/src/glusterd-tierd-svc.c b/xlators/mgmt/glusterd/src/glusterd-tierd-svc.c index 04ceec5d85e..ab463f19425 100644 --- a/xlators/mgmt/glusterd/src/glusterd-tierd-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-tierd-svc.c @@ -83,7 +83,6 @@ glusterd_tierdsvc_init(void *data) goto out; notify = glusterd_svc_common_rpc_notify; - glusterd_store_perform_node_state_store(volinfo); volinfo->type = GF_CLUSTER_TYPE_TIER; @@ -395,6 +394,7 @@ int glusterd_tierdsvc_restart() { glusterd_volinfo_t *volinfo = NULL; + glusterd_volinfo_t *tmp = NULL; int ret = 0; xlator_t *this = THIS; glusterd_conf_t *conf = NULL; @@ -405,7 +405,7 @@ glusterd_tierdsvc_restart() conf = this->private; GF_VALIDATE_OR_GOTO(this->name, conf, out); - cds_list_for_each_entry(volinfo, &conf->volumes, vol_list) + cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list) { /* Start per volume tierd svc */ if (volinfo->status == GLUSTERD_STATUS_STARTED && diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index e95bf9665c9..3570b018eeb 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -61,6 +61,7 @@ #include "glusterd-server-quorum.h" #include <glusterfs/quota-common-utils.h> #include <glusterfs/common-utils.h> +#include "glusterd-shd-svc-helper.h" #include "xdr-generic.h" #include <sys/resource.h> @@ -580,13 +581,17 @@ glusterd_volinfo_t * glusterd_volinfo_unref(glusterd_volinfo_t *volinfo) { int refcnt = -1; + glusterd_conf_t *conf = THIS->private; - pthread_mutex_lock(&volinfo->reflock); + pthread_mutex_lock(&conf->volume_lock); { - refcnt = --volinfo->refcnt; + pthread_mutex_lock(&volinfo->reflock); + { + refcnt = --volinfo->refcnt; + } + pthread_mutex_unlock(&volinfo->reflock); } - pthread_mutex_unlock(&volinfo->reflock); - + pthread_mutex_unlock(&conf->volume_lock); if (!refcnt) { glusterd_volinfo_delete(volinfo); return NULL; @@ -658,6 +663,7 @@ glusterd_volinfo_new(glusterd_volinfo_t **volinfo) glusterd_snapdsvc_build(&new_volinfo->snapd.svc); glusterd_tierdsvc_build(&new_volinfo->tierd.svc); glusterd_gfproxydsvc_build(&new_volinfo->gfproxyd.svc); + glusterd_shdsvc_build(&new_volinfo->shd.svc); pthread_mutex_init(&new_volinfo->reflock, NULL); *volinfo = glusterd_volinfo_ref(new_volinfo); @@ -1023,11 +1029,11 @@ glusterd_volinfo_delete(glusterd_volinfo_t *volinfo) gf_store_handle_destroy(volinfo->snapd.handle); glusterd_auth_cleanup(volinfo); + glusterd_shd_svcproc_cleanup(&volinfo->shd); pthread_mutex_destroy(&volinfo->reflock); GF_FREE(volinfo); ret = 0; - out: gf_msg_debug(THIS->name, 0, "Returning %d", ret); return ret; @@ -3553,6 +3559,7 @@ glusterd_spawn_daemons(void *opaque) ret = glusterd_snapdsvc_restart(); ret = glusterd_tierdsvc_restart(); ret = glusterd_gfproxydsvc_restart(); + ret = glusterd_shdsvc_restart(); return ret; } @@ -4503,6 +4510,9 @@ glusterd_delete_stale_volume(glusterd_volinfo_t *stale_volinfo, svc = &(stale_volinfo->snapd.svc); (void)svc->manager(svc, stale_volinfo, PROC_START_NO_WAIT); } + svc = &(stale_volinfo->shd.svc); + (void)svc->manager(svc, stale_volinfo, PROC_START_NO_WAIT); + (void)glusterd_volinfo_remove(stale_volinfo); return 0; @@ -4617,6 +4627,15 @@ glusterd_import_friend_volume(dict_t *peer_data, int count) glusterd_volinfo_unref(old_volinfo); } + ret = glusterd_store_volinfo(new_volinfo, GLUSTERD_VOLINFO_VER_AC_NONE); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL, + "Failed to store " + "volinfo for volume %s", + new_volinfo->volname); + goto out; + } + if (glusterd_is_volume_started(new_volinfo)) { (void)glusterd_start_bricks(new_volinfo); if (glusterd_is_snapd_enabled(new_volinfo)) { @@ -4625,15 +4644,10 @@ glusterd_import_friend_volume(dict_t *peer_data, int count) gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name); } } - } - - ret = glusterd_store_volinfo(new_volinfo, GLUSTERD_VOLINFO_VER_AC_NONE); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL, - "Failed to store " - "volinfo for volume %s", - new_volinfo->volname); - goto out; + svc = &(new_volinfo->shd.svc); + if (svc->manager(svc, new_volinfo, PROC_START_NO_WAIT)) { + gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name); + } } ret = glusterd_create_volfiles_and_notify_services(new_volinfo); @@ -5108,9 +5122,7 @@ glusterd_add_node_to_dict(char *server, dict_t *dict, int count, glusterd_svc_build_pidfile_path(server, priv->rundir, pidfile, sizeof(pidfile)); - if (strcmp(server, priv->shd_svc.name) == 0) - svc = &(priv->shd_svc); - else if (strcmp(server, priv->nfs_svc.name) == 0) + if (strcmp(server, priv->nfs_svc.name) == 0) svc = &(priv->nfs_svc); else if (strcmp(server, priv->quotad_svc.name) == 0) svc = &(priv->quotad_svc); @@ -5141,9 +5153,6 @@ glusterd_add_node_to_dict(char *server, dict_t *dict, int count, if (!strcmp(server, priv->nfs_svc.name)) ret = dict_set_nstrn(dict, key, keylen, "NFS Server", SLEN("NFS Server")); - else if (!strcmp(server, priv->shd_svc.name)) - ret = dict_set_nstrn(dict, key, keylen, "Self-heal Daemon", - SLEN("Self-heal Daemon")); else if (!strcmp(server, priv->quotad_svc.name)) ret = dict_set_nstrn(dict, key, keylen, "Quota Daemon", SLEN("Quota Daemon")); @@ -8709,6 +8718,21 @@ glusterd_friend_remove_cleanup_vols(uuid_t uuid) "to stop snapd daemon service"); } } + + if (glusterd_is_shd_compatible_volume(volinfo)) { + /* + * Sending stop request for all volumes. So it is fine + * to send stop for mux shd + */ + svc = &(volinfo->shd.svc); + ret = svc->stop(svc, SIGTERM); + if (ret) { + gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL, + "Failed " + "to stop shd daemon service"); + } + } + if (volinfo->type == GF_CLUSTER_TYPE_TIER) { svc = &(volinfo->tierd.svc); ret = svc->stop(svc, SIGTERM); @@ -8734,7 +8758,7 @@ glusterd_friend_remove_cleanup_vols(uuid_t uuid) } /* Reconfigure all daemon services upon peer detach */ - ret = glusterd_svcs_reconfigure(); + ret = glusterd_svcs_reconfigure(NULL); if (ret) { gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL, "Failed to reconfigure all daemon services."); @@ -14286,3 +14310,74 @@ glusterd_is_profile_on(glusterd_volinfo_t *volinfo) return _gf_true; return _gf_false; } + +int32_t +glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, + int32_t count) +{ + int ret = -1; + int32_t pid = -1; + int32_t brick_online = -1; + char key[64] = {0}; + int keylen; + char *pidfile = NULL; + xlator_t *this = NULL; + char *uuid_str = NULL; + + this = THIS; + GF_VALIDATE_OR_GOTO(THIS->name, this, out); + + GF_VALIDATE_OR_GOTO(this->name, volinfo, out); + GF_VALIDATE_OR_GOTO(this->name, dict, out); + + keylen = snprintf(key, sizeof(key), "brick%d.hostname", count); + ret = dict_set_nstrn(dict, key, keylen, "Self-heal Daemon", + SLEN("Self-heal Daemon")); + if (ret) + goto out; + + keylen = snprintf(key, sizeof(key), "brick%d.path", count); + uuid_str = gf_strdup(uuid_utoa(MY_UUID)); + if (!uuid_str) { + ret = -1; + goto out; + } + ret = dict_set_dynstrn(dict, key, keylen, uuid_str); + if (ret) + goto out; + uuid_str = NULL; + + /* shd doesn't have a port. but the cli needs a port key with + * a zero value to parse. + * */ + + keylen = snprintf(key, sizeof(key), "brick%d.port", count); + ret = dict_set_int32n(dict, key, keylen, 0); + if (ret) + goto out; + + pidfile = volinfo->shd.svc.proc.pidfile; + + brick_online = gf_is_service_running(pidfile, &pid); + + /* If shd is not running, then don't print the pid */ + if (!brick_online) + pid = -1; + keylen = snprintf(key, sizeof(key), "brick%d.pid", count); + ret = dict_set_int32n(dict, key, keylen, pid); + if (ret) + goto out; + + keylen = snprintf(key, sizeof(key), "brick%d.status", count); + ret = dict_set_int32n(dict, key, keylen, brick_online); + +out: + if (uuid_str) + GF_FREE(uuid_str); + if (ret) + gf_msg(this ? this->name : "glusterd", GF_LOG_ERROR, 0, + GD_MSG_DICT_SET_FAILED, + "Returning %d. adding values to dict failed", ret); + + return ret; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index 9bf19a67528..3647c343b47 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -876,4 +876,8 @@ glusterd_is_profile_on(glusterd_volinfo_t *volinfo); char * search_brick_path_from_proc(pid_t brick_pid, char *brickpath); + +int32_t +glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, + int32_t count); #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index a0a9c3b6644..9f3496b03e2 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -36,6 +36,7 @@ #include "glusterd-svc-mgmt.h" #include "glusterd-svc-helper.h" #include "glusterd-snapd-svc-helper.h" +#include "glusterd-shd-svc-helper.h" #include "glusterd-gfproxyd-svc-helper.h" struct gd_validate_reconf_opts { @@ -4780,7 +4781,7 @@ volgen_get_shd_key(int type) static int volgen_set_shd_key_enable(dict_t *set_dict, const int type) { - int ret = -1; + int ret = 0; switch (type) { case GF_CLUSTER_TYPE_REPLICATE: @@ -5064,24 +5065,15 @@ out: static int build_shd_volume_graph(xlator_t *this, volgen_graph_t *graph, glusterd_volinfo_t *volinfo, dict_t *mod_dict, - dict_t *set_dict, gf_boolean_t graph_check, - gf_boolean_t *valid_config) + dict_t *set_dict, gf_boolean_t graph_check) { volgen_graph_t cgraph = {0}; int ret = 0; int clusters = -1; - if (!graph_check && (volinfo->status != GLUSTERD_STATUS_STARTED)) - goto out; - if (!glusterd_is_shd_compatible_volume(volinfo)) goto out; - /* Shd graph is valid only when there is at least one - * replica/disperse volume is present - */ - *valid_config = _gf_true; - ret = prepare_shd_volume_options(volinfo, mod_dict, set_dict); if (ret) goto out; @@ -5111,19 +5103,16 @@ out: } int -build_shd_graph(volgen_graph_t *graph, dict_t *mod_dict) +build_shd_graph(glusterd_volinfo_t *volinfo, volgen_graph_t *graph, + dict_t *mod_dict) { - glusterd_volinfo_t *voliter = NULL; xlator_t *this = NULL; - glusterd_conf_t *priv = NULL; dict_t *set_dict = NULL; int ret = 0; - gf_boolean_t valid_config = _gf_false; xlator_t *iostxl = NULL; gf_boolean_t graph_check = _gf_false; this = THIS; - priv = this->private; set_dict = dict_new(); if (!set_dict) { @@ -5133,26 +5122,18 @@ build_shd_graph(volgen_graph_t *graph, dict_t *mod_dict) if (mod_dict) graph_check = dict_get_str_boolean(mod_dict, "graph-check", 0); - iostxl = volgen_graph_add_as(graph, "debug/io-stats", "glustershd"); + iostxl = volgen_graph_add_as(graph, "debug/io-stats", volinfo->volname); if (!iostxl) { ret = -1; goto out; } - cds_list_for_each_entry(voliter, &priv->volumes, vol_list) - { - ret = build_shd_volume_graph(this, graph, voliter, mod_dict, set_dict, - graph_check, &valid_config); - ret = dict_reset(set_dict); - if (ret) - goto out; - } + ret = build_shd_volume_graph(this, graph, volinfo, mod_dict, set_dict, + graph_check); out: if (set_dict) dict_unref(set_dict); - if (!valid_config) - ret = -EINVAL; return ret; } @@ -6469,6 +6450,10 @@ glusterd_create_volfiles(glusterd_volinfo_t *volinfo) if (ret) gf_log(this->name, GF_LOG_ERROR, "Could not generate gfproxy volfiles"); + ret = glusterd_shdsvc_create_volfile(volinfo); + if (ret) + gf_log(this->name, GF_LOG_ERROR, "Could not generate shd volfiles"); + dict_del_sizen(volinfo->dict, "skip-CLIOT"); out: @@ -6549,7 +6534,7 @@ validate_shdopts(glusterd_volinfo_t *volinfo, dict_t *val_dict, ret = dict_set_int32_sizen(val_dict, "graph-check", 1); if (ret) goto out; - ret = build_shd_graph(&graph, val_dict); + ret = build_shd_graph(volinfo, &graph, val_dict); if (!ret) ret = graph_reconf_validateopt(&graph.graph, op_errstr); @@ -6926,3 +6911,22 @@ gd_is_boolean_option(char *key) return _gf_false; } + +int +glusterd_shdsvc_generate_volfile(glusterd_volinfo_t *volinfo, char *filename, + dict_t *mode_dict) +{ + int ret = -1; + volgen_graph_t graph = { + 0, + }; + + graph.type = GF_SHD; + ret = build_shd_graph(volinfo, &graph, mode_dict); + if (!ret) + ret = volgen_write_volfile(&graph, filename); + + volgen_graph_free(&graph); + + return ret; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h index 37eecc04bef..5c4bfe0db0b 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.h +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h @@ -69,6 +69,7 @@ typedef enum { GF_REBALANCED = 1, GF_QUOTAD, GF_SNAPD, + GF_SHD, } glusterd_graph_type_t; struct volgen_graph { @@ -80,6 +81,8 @@ typedef struct volgen_graph volgen_graph_t; typedef int (*glusterd_graph_builder_t)(volgen_graph_t *graph, dict_t *mod_dict); +typedef int (*glusterd_vol_graph_builder_t)(glusterd_volinfo_t *, + char *filename, dict_t *mod_dict); #define COMPLETE_OPTION(key, completion, ret) \ do { \ @@ -204,7 +207,8 @@ void glusterd_get_shd_filepath(char *filename); int -build_shd_graph(volgen_graph_t *graph, dict_t *mod_dict); +build_shd_graph(glusterd_volinfo_t *volinfo, volgen_graph_t *graph, + dict_t *mod_dict); int build_nfs_graph(volgen_graph_t *graph, dict_t *mod_dict); @@ -316,4 +320,9 @@ glusterd_generate_gfproxyd_volfile(glusterd_volinfo_t *volinfo); int glusterd_build_gfproxyd_volfile(glusterd_volinfo_t *volinfo, char *filename); + +int +glusterd_shdsvc_generate_volfile(glusterd_volinfo_t *volinfo, char *filename, + dict_t *mode_dict); + #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c index 317cb04dd00..f9c7a2d70e9 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c @@ -1799,7 +1799,7 @@ static int glusterd_handle_heal_cmd(xlator_t *this, glusterd_volinfo_t *volinfo, dict_t *dict, char **op_errstr) { - glusterd_conf_t *priv = NULL; + glusterd_svc_t *svc = NULL; gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID; int ret = 0; char msg[2408] = { @@ -1809,7 +1809,6 @@ glusterd_handle_heal_cmd(xlator_t *this, glusterd_volinfo_t *volinfo, "Self-heal daemon is not running. " "Check self-heal daemon log file."; - priv = this->private; ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"), (int32_t *)&heal_op); if (ret) { @@ -1818,6 +1817,7 @@ glusterd_handle_heal_cmd(xlator_t *this, glusterd_volinfo_t *volinfo, goto out; } + svc = &(volinfo->shd.svc); switch (heal_op) { case GF_SHD_OP_INVALID: case GF_SHD_OP_HEAL_ENABLE: /* This op should be handled in volume-set*/ @@ -1847,7 +1847,7 @@ glusterd_handle_heal_cmd(xlator_t *this, glusterd_volinfo_t *volinfo, goto out; } - if (!priv->shd_svc.online) { + if (!svc->online) { ret = -1; *op_errstr = gf_strdup(offline_msg); goto out; @@ -1868,7 +1868,7 @@ glusterd_handle_heal_cmd(xlator_t *this, glusterd_volinfo_t *volinfo, goto out; } - if (!priv->shd_svc.online) { + if (!svc->online) { ret = -1; *op_errstr = gf_strdup(offline_msg); goto out; diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c index 280c9f3640e..d4ab63095e2 100644 --- a/xlators/mgmt/glusterd/src/glusterd.c +++ b/xlators/mgmt/glusterd/src/glusterd.c @@ -1537,14 +1537,6 @@ init(xlator_t *this) exit(1); } - ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_GLUSTERSHD_RUN_DIR); - if (ret) { - gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED, - "Unable to create " - "glustershd running directory"); - exit(1); - } - ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_NFS_RUN_DIR); if (ret) { gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED, @@ -1819,6 +1811,9 @@ init(xlator_t *this) CDS_INIT_LIST_HEAD(&conf->snapshots); CDS_INIT_LIST_HEAD(&conf->missed_snaps_list); CDS_INIT_LIST_HEAD(&conf->brick_procs); + CDS_INIT_LIST_HEAD(&conf->shd_procs); + pthread_mutex_init(&conf->attach_lock, NULL); + pthread_mutex_init(&conf->volume_lock, NULL); pthread_mutex_init(&conf->mutex, NULL); conf->rpc = rpc; @@ -1899,7 +1894,6 @@ init(xlator_t *this) glusterd_mgmt_v3_lock_timer_init(); glusterd_txn_opinfo_dict_init(); - glusterd_shdsvc_build(&conf->shd_svc); glusterd_nfssvc_build(&conf->nfs_svc); glusterd_quotadsvc_build(&conf->quotad_svc); glusterd_bitdsvc_build(&conf->bitd_svc); diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 0a2fffdbe63..eb89e3be93e 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -28,6 +28,7 @@ #include "glusterd-sm.h" #include "glusterd-snapd-svc.h" #include "glusterd-tierd-svc.h" +#include "glusterd-shd-svc.h" #include "glusterd-bitd-svc.h" #include "glusterd1-xdr.h" #include "protocol-common.h" @@ -167,7 +168,6 @@ typedef struct { char workdir[VALID_GLUSTERD_PATHMAX]; char rundir[VALID_GLUSTERD_PATHMAX]; rpcsvc_t *rpc; - glusterd_svc_t shd_svc; glusterd_svc_t nfs_svc; glusterd_svc_t bitd_svc; glusterd_svc_t scrub_svc; @@ -176,6 +176,7 @@ typedef struct { struct cds_list_head volumes; struct cds_list_head snapshots; /*List of snap volumes */ struct cds_list_head brick_procs; /* List of brick processes */ + struct cds_list_head shd_procs; /* List of shd processes */ pthread_mutex_t xprt_lock; struct list_head xprt_list; pthread_mutex_t import_volumes; @@ -216,6 +217,11 @@ typedef struct { gf_atomic_t blockers; uint32_t mgmt_v3_lock_timeout; gf_boolean_t restart_bricks; + pthread_mutex_t attach_lock; /* Lock can be per process or a common one */ + pthread_mutex_t volume_lock; /* We release the big_lock from lot of places + which might lead the modification of volinfo + list. + */ } glusterd_conf_t; typedef enum gf_brick_status { @@ -495,6 +501,7 @@ struct glusterd_volinfo_ { glusterd_snapdsvc_t snapd; glusterd_tierdsvc_t tierd; + glusterd_shdsvc_t shd; glusterd_gfproxydsvc_t gfproxyd; int32_t quota_xattr_version; gf_boolean_t stage_deleted; /* volume has passed staging @@ -621,7 +628,6 @@ typedef enum { #define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR "/gluster/snaps" #define GLUSTERD_BITD_RUN_DIR "/bitd" #define GLUSTERD_SCRUB_RUN_DIR "/scrub" -#define GLUSTERD_GLUSTERSHD_RUN_DIR "/glustershd" #define GLUSTERD_NFS_RUN_DIR "/nfs" #define GLUSTERD_QUOTAD_RUN_DIR "/quotad" #define GLUSTER_SHARED_STORAGE_BRICK_DIR GLUSTERD_DEFAULT_WORKDIR "/ss_brick" @@ -677,6 +683,26 @@ typedef ssize_t (*gd_serialize_t)(struct iovec outmsg, void *args); } \ } while (0) +#define GLUSTERD_GET_SHD_RUNDIR(path, volinfo, priv) \ + do { \ + int32_t _shd_dir_len; \ + _shd_dir_len = snprintf(path, PATH_MAX, "%s/shd/%s", priv->rundir, \ + volinfo->volname); \ + if ((_shd_dir_len < 0) || (_shd_dir_len >= PATH_MAX)) { \ + path[0] = 0; \ + } \ + } while (0) + +#define GLUSTERD_GET_SHD_PID_FILE(path, volinfo, priv) \ + do { \ + int32_t _shd_pid_len; \ + _shd_pid_len = snprintf(path, PATH_MAX, "%s/shd/%s-shd.pid", \ + priv->rundir, volinfo->volname); \ + if ((_shd_pid_len < 0) || (_shd_pid_len >= PATH_MAX)) { \ + path[0] = 0; \ + } \ + } while (0) + #define GLUSTERD_GET_VOLUME_PID_DIR(path, volinfo, priv) \ do { \ int32_t _vol_pid_len; \ diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c index 5aa3d9418f6..1e9f47297ee 100644 --- a/xlators/protocol/client/src/client.c +++ b/xlators/protocol/client/src/client.c @@ -46,7 +46,6 @@ client_fini_complete(xlator_t *this) GF_VALIDATE_OR_GOTO(this->name, this->private, out); clnt_conf_t *conf = this->private; - if (!conf->destroy) return 0; @@ -69,6 +68,11 @@ client_notify_dispatch_uniq(xlator_t *this, int32_t event, void *data, ...) return 0; return client_notify_dispatch(this, event, data); + + /* Please avoid any code that access xlator object here + * Because for a child down event, once we do the signal + * we will start cleanup. + */ } int @@ -105,6 +109,11 @@ client_notify_dispatch(xlator_t *this, int32_t event, void *data, ...) } pthread_mutex_unlock(&ctx->notify_lock); + /* Please avoid any code that access xlator object here + * Because for a child down event, once we do the signal + * we will start cleanup. + */ + return ret; } @@ -2272,6 +2281,7 @@ client_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, { xlator_t *this = NULL; clnt_conf_t *conf = NULL; + gf_boolean_t is_parent_down = _gf_false; int ret = 0; this = mydata; @@ -2333,6 +2343,19 @@ client_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, if (conf->portmap_err_logged) conf->disconnect_err_logged = 1; } + /* + * Once we complete the child down notification, + * There is a chance that the graph might get freed, + * So it is not safe to access any xlator contens + * So here we are checking whether the parent is down + * or not. + */ + pthread_mutex_lock(&conf->lock); + { + is_parent_down = conf->parent_down; + } + pthread_mutex_unlock(&conf->lock); + /* If the CHILD_DOWN event goes to parent xlator multiple times, the logic of parent xlator notify may get screwed up.. (eg. CHILD_MODIFIED event in @@ -2340,6 +2363,12 @@ client_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, to parent are genuine */ ret = client_notify_dispatch_uniq(this, GF_EVENT_CHILD_DOWN, NULL); + if (is_parent_down) { + /* If parent is down, then there should not be any + * operation after a child down. + */ + goto out; + } if (ret) gf_msg(this->name, GF_LOG_INFO, 0, PC_MSG_CHILD_DOWN_NOTIFY_FAILED, |