diff options
Diffstat (limited to 'xlators/protocol/server/src/server.c')
| -rw-r--r-- | xlators/protocol/server/src/server.c | 526 |
1 files changed, 316 insertions, 210 deletions
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index b9879cdf148..721968004a0 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -14,15 +14,14 @@ #include "server.h" #include "server-helpers.h" #include "glusterfs3-xdr.h" -#include "call-stub.h" -#include "statedump.h" -#include "defaults.h" +#include <glusterfs/call-stub.h> +#include <glusterfs/statedump.h> +#include <glusterfs/defaults.h> #include "authenticate.h" -#include "gf-event.h" -#include "events.h" +#include <glusterfs/gf-event.h> +#include <glusterfs/events.h> #include "server-messages.h" #include "rpc-clnt.h" -#include "glusterfsd.h" rpcsvc_cbk_program_t server_cbk_prog = { .progname = "Gluster Callback", @@ -109,8 +108,7 @@ server_submit_reply(call_frame_t *frame, rpcsvc_request_t *req, void *arg, iob = gfs_serialize_reply(req, arg, &rsp, xdrproc); if (!iob) { - gf_msg("", GF_LOG_ERROR, 0, PS_MSG_SERIALIZE_REPLY_FAILED, - "Failed to serialize reply"); + gf_smsg("", GF_LOG_ERROR, 0, PS_MSG_SERIALIZE_REPLY_FAILED, NULL); goto ret; } @@ -133,7 +131,7 @@ server_submit_reply(call_frame_t *frame, rpcsvc_request_t *req, void *arg, "Reply submission failed"); if (frame && client) { server_connection_cleanup(frame->this, client, - INTERNAL_LOCKS | POSIX_LOCKS); + INTERNAL_LOCKS | POSIX_LOCKS, NULL); } else { gf_msg_callingfn("", GF_LOG_ERROR, 0, PS_MSG_REPLY_SUBMIT_FAILED, "Reply submission failed"); @@ -169,6 +167,7 @@ server_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname) char key[32] = { 0, }; + int keylen; int count = 0; int ret = -1; @@ -188,8 +187,8 @@ server_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname) (xprt->xl_private->bound_xl->name) && (brickname) && (!strcmp(brickname, xprt->xl_private->bound_xl->name))) { peerinfo = &xprt->peerinfo; - snprintf(key, sizeof(key), "client%d.hostname", count); - ret = dict_set_str(dict, key, peerinfo->identifier); + keylen = snprintf(key, sizeof(key), "client%d.hostname", count); + ret = dict_set_strn(dict, key, keylen, peerinfo->identifier); if (ret) goto unlock; @@ -208,8 +207,9 @@ server_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname) if (ret) goto unlock; - snprintf(key, sizeof(key), "client%d.name", count); - ret = dict_set_str(dict, key, xprt->xl_private->client_name); + keylen = snprintf(key, sizeof(key), "client%d.name", count); + ret = dict_set_strn(dict, key, keylen, + xprt->xl_private->client_name); if (ret) goto unlock; @@ -222,7 +222,7 @@ unlock: if (ret) goto out; - ret = dict_set_int32(dict, "clientcount", count); + ret = dict_set_int32_sizen(dict, "clientcount", count); out: return ret; @@ -247,7 +247,7 @@ server_priv(xlator_t *this) return 0; gf_proc_dump_build_key(key, "xlator.protocol.server", "priv"); - gf_proc_dump_add_section(key); + gf_proc_dump_add_section("%s", key); ret = pthread_mutex_trylock(&conf->mutex); if (ret != 0) @@ -267,6 +267,8 @@ server_priv(xlator_t *this) gf_proc_dump_build_key(key, "server", "total-bytes-write"); gf_proc_dump_write(key, "%" PRIu64, total_write); + rpcsvc_statedump(conf->rpc); + ret = 0; out: if (ret) @@ -302,8 +304,7 @@ get_auth_types(dict_t *this, char *key, data_t *value, void *data) /* TODO: backward compatibility, remove when newer versions are available */ tmp = "addr"; - gf_msg("server", GF_LOG_WARNING, 0, PS_MSG_AUTH_IP_ERROR, - "assuming 'auth.ip' to be 'auth.addr'"); + gf_smsg("server", GF_LOG_WARNING, 0, PS_MSG_AUTH_IP_ERROR, NULL); } ret = dict_set_dynptr(auth_dict, tmp, NULL, 0); if (ret < 0) { @@ -332,8 +333,8 @@ _check_for_auth_option(dict_t *d, char *k, data_t *v, void *tmp) goto out; if (strncmp(tail, "addr.", 5) != 0) { - gf_msg(xl->name, GF_LOG_TRACE, 0, PS_MSG_SKIP_FORMAT_CHK, - "skip format check for non-addr auth option %s", k); + gf_smsg(xl->name, GF_LOG_TRACE, 0, PS_MSG_SKIP_FORMAT_CHK, "option=%s", + k, NULL); goto out; } @@ -355,10 +356,8 @@ _check_for_auth_option(dict_t *d, char *k, data_t *v, void *tmp) ret = xlator_option_validate_addr_list(xl, "auth-*", v->data, NULL, NULL); if (ret) - gf_msg(xl->name, GF_LOG_ERROR, 0, PS_MSG_INTERNET_ADDR_ERROR, - "internet address '%s' does not conform " - "to standards.", - v->data); + gf_smsg(xl->name, GF_LOG_ERROR, 0, PS_MSG_INTERNET_ADDR_ERROR, + "data=%s", v->data, NULL); } out: return ret; @@ -378,11 +377,8 @@ validate_auth_options(xlator_t *this, dict_t *dict) error = dict_foreach(dict, _check_for_auth_option, trav->xlator); if (-1 == error) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_AUTHENTICATE_ERROR, - "volume '%s' " - "defined as subvolume, but no authentication " - "defined for the same", - trav->xlator->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_AUTHENTICATE_ERROR, + "name=%s", trav->xlator->name, NULL); break; } trav = trav->next; @@ -392,6 +388,43 @@ out: return error; } +void +server_call_xlator_mem_cleanup(xlator_t *this, char *victim_name) +{ + pthread_t th_id = { + 0, + }; + int th_ret = -1; + server_cleanup_xprt_arg_t *arg = NULL; + + if (!victim_name) + return; + + gf_log(this->name, GF_LOG_INFO, "Create graph janitor thread for brick %s", + victim_name); + + arg = calloc(1, sizeof(*arg)); + arg->this = this; + arg->victim_name = strdup(victim_name); + if (!arg->victim_name) { + gf_smsg(this->name, GF_LOG_CRITICAL, ENOMEM, LG_MSG_NO_MEMORY, + "Memory allocation is failed"); + free(arg); + return; + } + + th_ret = gf_thread_create_detached(&th_id, server_graph_janitor_threads, + arg, "graphjanitor"); + if (th_ret) { + gf_log(this->name, GF_LOG_ERROR, + "graph janitor Thread" + " creation is failed for brick %s", + victim_name); + free(arg->victim_name); + free(arg); + } +} + int server_rpc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, void *data) { @@ -402,25 +435,20 @@ server_rpc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, void *data) client_t *client = NULL; char *auth_path = NULL; int ret = -1; - gf_boolean_t victim_found = _gf_false; char *xlator_name = NULL; - glusterfs_ctx_t *ctx = NULL; - xlator_t *top = NULL; - xlator_list_t **trav_p = NULL; - xlator_t *travxl = NULL; uint64_t xprtrefcount = 0; - struct _child_status *tmp = NULL; + gf_boolean_t fd_exist = _gf_false; - if (!xl || !data) { + this = xl; + trans = data; + + if (!this || !data || !this->ctx || !this->ctx->active) { gf_msg_callingfn("server", GF_LOG_WARNING, 0, PS_MSG_RPC_NOTIFY_ERROR, "Calling rpc_notify without initializing"); goto out; } - this = xl; - trans = data; conf = this->private; - ctx = this->ctx; switch (event) { case RPCSVC_EVENT_ACCEPT: { @@ -451,35 +479,38 @@ server_rpc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, void *data) break; } + /* Set the disconnect_progress flag to 1 to avoid races + during brick detach while brick mux is enabled + */ + GF_ATOMIC_INIT(trans->disconnect_progress, 1); /* transport has to be removed from the list upon disconnect * irrespective of whether lock self heal is off or on, since * new transport will be created upon reconnect. */ pthread_mutex_lock(&conf->mutex); client = trans->xl_private; - list_del_init(&trans->list); + if (!client) + list_del_init(&trans->list); pthread_mutex_unlock(&conf->mutex); if (!client) goto unref_transport; - gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_CLIENT_DISCONNECTING, - "disconnecting connection" - " from %s", - client->client_uid); + gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_CLIENT_DISCONNECTING, + "client-uid=%s", client->client_uid, NULL); - ret = dict_get_str(this->options, "auth-path", &auth_path); + ret = dict_get_str_sizen(this->options, "auth-path", &auth_path); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_DICT_GET_FAILED, - "failed to get auth-path"); + gf_smsg(this->name, GF_LOG_WARNING, 0, PS_MSG_DICT_GET_FAILED, + "type=auth-path", NULL); auth_path = NULL; } gf_client_ref(client); gf_client_put(client, &detached); if (detached) { - server_connection_cleanup(this, client, - INTERNAL_LOCKS | POSIX_LOCKS); + server_connection_cleanup( + this, client, INTERNAL_LOCKS | POSIX_LOCKS, &fd_exist); gf_event(EVENT_CLIENT_DISCONNECT, "client_uid=%s;" "client_identifier=%s;server_identifier=%s;" @@ -496,53 +527,36 @@ server_rpc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, void *data) unref_transport: /* rpc_transport_unref() causes a RPCSVC_EVENT_TRANSPORT_DESTROY * to be called in blocking manner - * So no code should ideally be after this unref + * So no code should ideally be after this unref, Call + * rpc_transport_unref only while cleanup_starting flag is not set + * otherwise transport_unref will be call by either + * server_connection_cleanup_flush_cbk or server_submit_reply at the + * time of freeing state */ - rpc_transport_unref(trans); + if (!client || !detached || !fd_exist) + rpc_transport_unref(trans); break; case RPCSVC_EVENT_TRANSPORT_DESTROY: + pthread_mutex_lock(&conf->mutex); client = trans->xl_private; + list_del_init(&trans->list); + pthread_mutex_unlock(&conf->mutex); if (!client) break; - pthread_mutex_lock(&conf->mutex); - list_for_each_entry(tmp, &conf->child_status->status_list, - status_list) - { - if (tmp->name && client->bound_xl && - client->bound_xl->cleanup_starting && - !strcmp(tmp->name, client->bound_xl->name)) { - xprtrefcount = GF_ATOMIC_GET(tmp->xprtrefcnt); - if (xprtrefcount > 0) { - xprtrefcount = GF_ATOMIC_DEC(tmp->xprtrefcnt); - if (xprtrefcount == 0) - xlator_name = gf_strdup(client->bound_xl->name); - } - break; + + if (client->bound_xl && client->bound_xl->cleanup_starting) { + xprtrefcount = GF_ATOMIC_GET(client->bound_xl->xprtrefcnt); + if (xprtrefcount > 0) { + xprtrefcount = GF_ATOMIC_DEC(client->bound_xl->xprtrefcnt); + if (xprtrefcount == 0) + xlator_name = gf_strdup(client->bound_xl->name); } } - pthread_mutex_unlock(&conf->mutex); gf_client_unref(client); if (xlator_name) { - if (this->ctx->active) { - top = this->ctx->active->first; - LOCK(&ctx->volfile_lock); - for (trav_p = &top->children; *trav_p; - trav_p = &(*trav_p)->next) { - travxl = (*trav_p)->xlator; - if (!travxl->call_cleanup && - strcmp(travxl->name, xlator_name) == 0) { - victim_found = _gf_true; - break; - } - } - UNLOCK(&ctx->volfile_lock); - if (victim_found) { - xlator_mem_cleanup(travxl); - rpcsvc_autoscale_threads(ctx, conf->rpc, -1); - } - } + server_call_xlator_mem_cleanup(this, xlator_name); GF_FREE(xlator_name); } @@ -556,6 +570,71 @@ out: return 0; } +void * +server_graph_janitor_threads(void *data) +{ + xlator_t *victim = NULL; + xlator_t *this = NULL; + server_conf_t *conf = NULL; + glusterfs_ctx_t *ctx = NULL; + char *victim_name = NULL; + server_cleanup_xprt_arg_t *arg = NULL; + gf_boolean_t victim_found = _gf_false; + xlator_list_t **trav_p = NULL; + xlator_t *top = NULL; + uint32_t parent_down = 0; + + GF_ASSERT(data); + + arg = data; + this = arg->this; + victim_name = arg->victim_name; + THIS = arg->this; + conf = this->private; + + ctx = THIS->ctx; + GF_VALIDATE_OR_GOTO(this->name, ctx, out); + + top = this->ctx->active->first; + LOCK(&ctx->volfile_lock); + for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) { + victim = (*trav_p)->xlator; + if (victim->cleanup_starting && + strcmp(victim->name, victim_name) == 0) { + parent_down = victim->parent_down; + victim->parent_down = 1; + if (!parent_down) + victim_found = _gf_true; + break; + } + } + if (victim_found) + glusterfs_delete_volfile_checksum(ctx, victim->volfile_id); + UNLOCK(&ctx->volfile_lock); + if (!victim_found) { + gf_log(this->name, GF_LOG_ERROR, + "victim brick %s is not" + " found in graph", + victim_name); + goto out; + } + + default_notify(victim, GF_EVENT_PARENT_DOWN, victim); + if (victim->notify_down) { + gf_log(THIS->name, GF_LOG_INFO, + "Start call fini for brick" + " %s stack", + victim->name); + xlator_mem_cleanup(victim); + rpcsvc_autoscale_threads(ctx, conf->rpc, -1); + } + +out: + free(arg->victim_name); + free(arg); + return NULL; +} + int32_t server_mem_acct_init(xlator_t *this) { @@ -566,8 +645,7 @@ server_mem_acct_init(xlator_t *this) ret = xlator_mem_acct_init(this, gf_server_mt_end + 1); if (ret != 0) { - gf_msg(this->name, GF_LOG_ERROR, ENOMEM, PS_MSG_NO_MEMORY, - "Memory accounting init failed"); + gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, PS_MSG_NO_MEMORY, NULL); return ret; } out: @@ -623,7 +701,7 @@ server_check_event_threads(xlator_t *this, server_conf_t *conf, int32_t new) return 0; } - return event_reconfigure_threads(pool, target); + return gf_event_reconfigure_threads(pool, target); } int @@ -669,14 +747,15 @@ server_reconfigure(xlator_t *this, dict_t *options) * translator itself. */ kid = NULL; - if (dict_get_str(options, "auth-path", &auth_path) == 0) { + if (dict_get_str_sizen(options, "auth-path", &auth_path) == 0) { kid = get_xlator_by_name(this, auth_path); } if (!kid) { kid = this; } - if (dict_get_int32(options, "inode-lru-limit", &inode_lru_limit) == 0) { + if (dict_get_int32_sizen(options, "inode-lru-limit", &inode_lru_limit) == + 0) { conf->inode_lru_limit = inode_lru_limit; gf_msg_trace(this->name, 0, "Reconfigured inode-lru-limit to " @@ -691,13 +770,12 @@ server_reconfigure(xlator_t *this, dict_t *options) xlator_foreach(this, xlator_set_inode_lru_limit, &inode_lru_limit); } - data = dict_get(options, "trace"); + data = dict_get_sizen(options, "trace"); if (data) { ret = gf_string2boolean(data->data, &trace); if (ret != 0) { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, - "'trace' takes on only " - "boolean values. Neglecting option"); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, + NULL); ret = -1; goto out; } @@ -707,8 +785,7 @@ server_reconfigure(xlator_t *this, dict_t *options) GF_OPTION_RECONF("statedump-path", statedump_path, options, path, do_auth); if (!statedump_path) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_STATEDUMP_PATH_ERROR, - "Error while reconfiguring statedump path"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_STATEDUMP_PATH_ERROR, NULL); goto do_auth; } gf_path_strip_trailing_slashes(statedump_path); @@ -741,16 +818,14 @@ do_auth: GF_OPTION_RECONF("gid-timeout", conf->gid_cache_timeout, options, int32, do_rpc); if (gid_cache_reconf(&conf->gid_cache, conf->gid_cache_timeout) < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_GRP_CACHE_ERROR, - "Failed to reconfigure group cache."); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_GRP_CACHE_ERROR, NULL); goto do_rpc; } do_rpc: rpc_conf = conf->rpc; if (!rpc_conf) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONF_ERROR, - "No rpc_conf !!!!"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONF_ERROR, NULL); goto out; } @@ -794,8 +869,8 @@ do_rpc: * connections that are relevant to the brick * we're reconfiguring. */ - if (dict_get_str(xprt->clnt_options, "remote-subvolume", - &xprt_path) != 0) { + if (dict_get_str_sizen(xprt->clnt_options, "remote-subvolume", + &xprt_path) != 0) { continue; } if (strcmp(xprt_path, auth_path) != 0) { @@ -804,9 +879,8 @@ do_rpc: ret = gf_authenticate(xprt->clnt_options, options, conf->auth_modules); if (ret == AUTH_ACCEPT) { - gf_msg(kid->name, GF_LOG_TRACE, 0, PS_MSG_CLIENT_ACCEPTED, - "authorized client, hence we " - "continue with this connection"); + gf_smsg(kid->name, GF_LOG_TRACE, 0, PS_MSG_CLIENT_ACCEPTED, + NULL); } else { gf_event(EVENT_CLIENT_AUTH_REJECT, "client_uid=%s;" @@ -816,11 +890,10 @@ do_rpc: xprt->xl_private->client_uid, xprt->peerinfo.identifier, xprt->myinfo.identifier, auth_path); - gf_msg(this->name, GF_LOG_INFO, EACCES, - PS_MSG_AUTHENTICATE_ERROR, - "unauthorized client, hence " - "terminating the connection %s", - xprt->peerinfo.identifier); + gf_smsg(this->name, GF_LOG_INFO, EACCES, + PS_MSG_UNAUTHORIZED_CLIENT, + "peerinfo-identifier=%s", xprt->peerinfo.identifier, + NULL); rpc_transport_disconnect(xprt, _gf_false); } } @@ -831,8 +904,7 @@ do_rpc: ret = rpcsvc_set_outstanding_rpc_limit( rpc_conf, options, RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONF_ERROR, - "Failed to reconfigure outstanding-rpc-limit"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_RECONFIGURE_FAILED, NULL); goto out; } @@ -842,9 +914,8 @@ do_rpc: if (listeners->trans->reconfigure) listeners->trans->reconfigure(listeners->trans, options); else - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_ERROR, - "Reconfigure " - "not found for transport"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_ERROR, + NULL); } } @@ -860,12 +931,6 @@ do_rpc: if (ret) goto out; - /* rpcsvc thread reconfigure should be after events thread - * reconfigure - */ - new_nthread = ((struct event_pool *)(this->ctx->event_pool)) - ->eventthreadcount; - ret = rpcsvc_ownthread_reconf(rpc_conf, new_nthread); out: THIS = oldTHIS; gf_msg_debug("", 0, "returning %d", ret); @@ -910,9 +975,9 @@ server_dump_metrics(xlator_t *this, int fd) if (!client) continue; - dprintf(fd, "%s.total.rpc.%s.bytes_read %lu\n", this->name, + dprintf(fd, "%s.total.rpc.%s.bytes_read %" PRIu64 "\n", this->name, client->client_uid, xprt->total_bytes_read); - dprintf(fd, "%s.total.rpc.%s.bytes_write %lu\n", this->name, + dprintf(fd, "%s.total.rpc.%s.bytes_write %" PRIu64 "\n", this->name, client->client_uid, xprt->total_bytes_write); dprintf(fd, "%s.total.rpc.%s.outstanding %d\n", this->name, client->client_uid, xprt->outstanding_rpc_count); @@ -923,6 +988,49 @@ server_dump_metrics(xlator_t *this, int fd) return 0; } +void +server_cleanup(xlator_t *this, server_conf_t *conf) +{ + if (!this || !conf) + return; + + LOCK_DESTROY(&conf->itable_lock); + pthread_mutex_destroy(&conf->mutex); + + if (this->ctx->event_pool) { + /* Free the event pool */ + (void)gf_event_pool_destroy(this->ctx->event_pool); + } + + if (dict_get_sizen(this->options, "config-directory")) { + GF_FREE(conf->conf_dir); + conf->conf_dir = NULL; + } + + if (conf->child_status) { + GF_FREE(conf->child_status); + conf->child_status = NULL; + } + + if (this->ctx->statedump_path) { + GF_FREE(this->ctx->statedump_path); + this->ctx->statedump_path = NULL; + } + + if (conf->auth_modules) { + gf_auth_fini(conf->auth_modules); + dict_unref(conf->auth_modules); + } + + if (conf->rpc) { + (void)rpcsvc_destroy(conf->rpc); + conf->rpc = NULL; + } + + GF_FREE(conf); + this->private = NULL; +} + int server_init(xlator_t *this) { @@ -935,14 +1043,12 @@ server_init(xlator_t *this) GF_VALIDATE_OR_GOTO("init", this, out); if (this->children == NULL) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_SUBVOL_NULL, - "protocol/server should have subvolume"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_SUBVOL_NULL, NULL); goto out; } if (this->parents != NULL) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_PARENT_VOL_ERROR, - "protocol/server should not have parent volumes"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_PARENT_VOL_ERROR, NULL); goto out; } @@ -965,27 +1071,22 @@ server_init(xlator_t *this) if (ret) goto out; - ret = dict_get_str(this->options, "config-directory", &conf->conf_dir); + ret = dict_get_str_sizen(this->options, "config-directory", + &conf->conf_dir); if (ret) conf->conf_dir = CONFDIR; conf->child_status = GF_CALLOC(1, sizeof(struct _child_status), gf_server_mt_child_status); INIT_LIST_HEAD(&conf->child_status->status_list); - GF_ATOMIC_INIT(conf->child_status->xprtrefcnt, 0); - /*ret = dict_get_str (this->options, "statedump-path", &statedump_path); - if (!ret) { - gf_path_strip_trailing_slashes (statedump_path); - this->ctx->statedump_path = statedump_path; - }*/ GF_OPTION_INIT("statedump-path", statedump_path, path, out); if (statedump_path) { gf_path_strip_trailing_slashes(statedump_path); this->ctx->statedump_path = gf_strdup(statedump_path); } else { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_STATEDUMP_PATH_ERROR, - "Error setting statedump path"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_SET_STATEDUMP_PATH_ERROR, + NULL); ret = -1; goto out; } @@ -1004,6 +1105,7 @@ server_init(xlator_t *this) ret = gf_auth_init(this, conf->auth_modules); if (ret) { dict_unref(conf->auth_modules); + conf->auth_modules = NULL; goto out; } @@ -1015,8 +1117,7 @@ server_init(xlator_t *this) GF_OPTION_INIT("gid-timeout", conf->gid_cache_timeout, int32, out); if (gid_cache_init(&conf->gid_cache, conf->gid_cache_timeout) < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_GRP_CACHE_ERROR, - "Failed to initialize group cache."); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_INIT_GRP_CACHE_ERROR, NULL); goto out; } @@ -1035,9 +1136,7 @@ server_init(xlator_t *this) /* RPC related */ conf->rpc = rpcsvc_init(this, this->ctx, this->options, 0); if (conf->rpc == NULL) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPCSVC_CREATE_FAILED, - "creation of rpcsvc " - "failed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPCSVC_CREATE_FAILED, NULL); ret = -1; goto out; } @@ -1045,8 +1144,7 @@ server_init(xlator_t *this) ret = rpcsvc_set_outstanding_rpc_limit( conf->rpc, this->options, RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONF_ERROR, - "Failed to configure outstanding-rpc-limit"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONFIGURE_FAILED, NULL); goto out; } @@ -1056,40 +1154,40 @@ server_init(xlator_t *this) */ this->ctx->secure_srvr = MGMT_SSL_COPY_IO; - ret = dict_get_str(this->options, "transport-type", &transport_type); + ret = dict_get_str_sizen(this->options, "transport-type", &transport_type); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_ERROR, - "option transport-type not set"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_TYPE_NOT_SET, + NULL); ret = -1; goto out; } total_transport = rpc_transport_count(transport_type); if (total_transport <= 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_ERROR, - "failed to get total number of available tranpsorts"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_GET_TOTAL_AVAIL_TRANSPORT_FAILED, NULL); ret = -1; goto out; } + + ret = dict_set_int32_sizen(this->options, "notify-poller-death", 1); + ret = rpcsvc_create_listeners(conf->rpc, this->options, this->name); if (ret < 1) { - gf_msg(this->name, GF_LOG_WARNING, 0, - PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, - "creation of listener failed"); + gf_smsg(this->name, GF_LOG_WARNING, 0, + PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, NULL); if (ret != -EADDRINUSE) ret = -1; goto out; } else if (ret < total_transport) { - gf_msg(this->name, GF_LOG_ERROR, 0, - PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, - "creation of %d listeners failed, continuing with " - "succeeded transport", - (total_transport - ret)); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, "number=%d", + "continuing with succeeded transport", (total_transport - ret), + NULL); } ret = rpcsvc_register_notify(conf->rpc, server_rpc_notify, this); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_RPCSVC_NOTIFY, - "registration of notify with rpcsvc failed"); + gf_smsg(this->name, GF_LOG_WARNING, 0, PS_MSG_RPCSVC_NOTIFY, NULL); goto out; } @@ -1099,11 +1197,10 @@ server_init(xlator_t *this) */ ret = rpcsvc_program_register(conf->rpc, &glusterfs3_3_fop_prog, _gf_true); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_PGM_REG_FAILED, - "registration of program (name:%s, prognum:%d, " - "progver:%d) failed", - glusterfs3_3_fop_prog.progname, glusterfs3_3_fop_prog.prognum, - glusterfs3_3_fop_prog.progver); + gf_smsg(this->name, GF_LOG_WARNING, 0, PS_MSG_PGM_REG_FAILED, "name=%s", + glusterfs3_3_fop_prog.progname, "prognum=%d", + glusterfs3_3_fop_prog.prognum, "progver=%d", + glusterfs3_3_fop_prog.progver, NULL); goto out; } @@ -1123,11 +1220,10 @@ server_init(xlator_t *this) ret = rpcsvc_program_register(conf->rpc, &gluster_handshake_prog, _gf_false); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_PGM_REG_FAILED, - "registration of program (name:%s, prognum:%d, " - "progver:%d) failed", - gluster_handshake_prog.progname, gluster_handshake_prog.prognum, - gluster_handshake_prog.progver); + gf_smsg(this->name, GF_LOG_WARNING, 0, PS_MSG_PGM_REG_FAILED, "name=%s", + gluster_handshake_prog.progname, "prognum=%d", + gluster_handshake_prog.prognum, "progver=%d", + gluster_handshake_prog.progver, NULL); rpcsvc_program_unregister(conf->rpc, &glusterfs3_3_fop_prog); rpcsvc_program_unregister(conf->rpc, &glusterfs4_0_fop_prog); goto out; @@ -1141,18 +1237,14 @@ server_init(xlator_t *this) lim.rlim_max = 1048576; if (setrlimit(RLIMIT_NOFILE, &lim) == -1) { - gf_msg(this->name, GF_LOG_WARNING, errno, PS_MSG_ULIMIT_SET_FAILED, - "WARNING: Failed to " - "set 'ulimit -n 1M': %s", - strerror(errno)); + gf_smsg(this->name, GF_LOG_WARNING, errno, PS_MSG_ULIMIT_SET_FAILED, + "errno=%s", strerror(errno), NULL); lim.rlim_cur = 65536; lim.rlim_max = 65536; if (setrlimit(RLIMIT_NOFILE, &lim) == -1) { - gf_msg(this->name, GF_LOG_WARNING, errno, PS_MSG_FD_NOT_FOUND, - "Failed to set " - "max open fd to 64k: %s", - strerror(errno)); + gf_smsg(this->name, GF_LOG_WARNING, errno, PS_MSG_FD_NOT_FOUND, + "errno=%s", strerror(errno), NULL); } else { gf_msg_trace(this->name, 0, "max open fd set " @@ -1164,9 +1256,8 @@ server_init(xlator_t *this) if (!this->ctx->cmd_args.volfile_id) { /* In some use cases this is a valid case, but document this to be annoying log in that case */ - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_VOL_FILE_OPEN_FAILED, - "volfile-id argument not given. " - "This is mandatory argument, defaulting to 'gluster'"); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_VOL_FILE_OPEN_FAILED, + NULL); this->ctx->cmd_args.volfile_id = gf_strdup("gluster"); } FIRST_CHILD(this)->volfile_id = gf_strdup(this->ctx->cmd_args.volfile_id); @@ -1179,15 +1270,7 @@ out: if (this != NULL) { this->fini(this); } - - if (conf && conf->rpc) { - rpcsvc_listener_t *listener, *next; - list_for_each_entry_safe(listener, next, &conf->rpc->listeners, - list) - { - rpcsvc_listener_destroy(listener); - } - } + server_cleanup(this, conf); } return ret; @@ -1310,9 +1393,9 @@ server_process_event_upcall(xlator_t *this, void *data) xdrproc = (xdrproc_t)xdr_gfs4_entrylk_contention_req; break; default: - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, - "Received invalid upcall event(%d)", - upcall_data->event_type); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, + PS_MSG_INVLAID_UPCALL_EVENT, "event-type=%d", + upcall_data->event_type, NULL); goto out; } @@ -1386,6 +1469,8 @@ server_process_child_event(xlator_t *this, int32_t event, void *data, INIT_LIST_HEAD(&tmp->status_list); tmp->name = gf_strdup(victim->name); tmp->child_up = _gf_true; + memcpy(tmp->volume_id, victim->graph->volume_id, + GF_UUID_BUF_SIZE); list_add_tail(&tmp->status_list, &conf->child_status->status_list); } @@ -1402,10 +1487,8 @@ server_process_child_event(xlator_t *this, int32_t event, void *data, } if (!tmp->name) - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_CHILD_STATUS_FAILED, - "No xlator %s is found in " - "child status list", - victim->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_CHILD_STATUS_FAILED, + "name=%s", victim->name, NULL); } list_for_each_entry(xprt, &conf->xprt_list, list) { @@ -1440,6 +1523,7 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) glusterfs_ctx_t *ctx = NULL; gf_boolean_t xprt_found = _gf_false; uint64_t totxprt = 0; + uint64_t totdisconnect = 0; GF_VALIDATE_OR_GOTO(THIS->name, this, out); conf = this->private; @@ -1453,9 +1537,8 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) ret = server_process_event_upcall(this, data); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, - PS_MSG_SERVER_EVENT_UPCALL_FAILED, - "server_process_event_upcall failed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_SERVER_EVENT_UPCALL_FAILED, NULL); goto out; } break; @@ -1474,9 +1557,8 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) ret = server_process_child_event(this, event, data, GF_CBK_CHILD_UP); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, - PS_MSG_SERVER_EVENT_UPCALL_FAILED, - "server_process_child_event failed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_SERVER_CHILD_EVENT_FAILED, NULL); goto out; } default_notify(this, event, data); @@ -1484,12 +1566,17 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) } case GF_EVENT_CHILD_DOWN: { + if (victim->cleanup_starting) { + victim->notify_down = 1; + gf_log(this->name, GF_LOG_INFO, + "Getting CHILD_DOWN event for brick %s", victim->name); + } + ret = server_process_child_event(this, event, data, GF_CBK_CHILD_DOWN); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, - PS_MSG_SERVER_EVENT_UPCALL_FAILED, - "server_process_child_event failed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_SERVER_CHILD_EVENT_FAILED, NULL); goto out; } default_notify(this, event, data); @@ -1507,6 +1594,10 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) if (!xprt->xl_private) { continue; } + + if (GF_ATOMIC_GET(xprt->disconnect_progress)) + continue; + if (xprt->xl_private->bound_xl == data) { totxprt++; } @@ -1517,7 +1608,7 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) { if (strcmp(tmp->name, victim->name) == 0) { tmp->child_up = _gf_false; - GF_ATOMIC_INIT(tmp->xprtrefcnt, totxprt); + GF_ATOMIC_INIT(victim->xprtrefcnt, totxprt); break; } } @@ -1533,14 +1624,22 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) if (!xprt->xl_private) { continue; } + + if (GF_ATOMIC_GET(xprt->disconnect_progress)) + continue; + if (xprt->xl_private->bound_xl == data) { gf_log(this->name, GF_LOG_INFO, "disconnecting %s", xprt->peerinfo.identifier); xprt_found = _gf_true; + totdisconnect++; rpc_transport_disconnect(xprt, _gf_false); } } + if (totxprt > totdisconnect) + GF_ATOMIC_SUB(victim->xprtrefcnt, (totxprt - totdisconnect)); + pthread_mutex_unlock(&conf->mutex); if (this->ctx->active) { top = this->ctx->active->first; @@ -1561,8 +1660,7 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) rpc_clnt_mgmt_pmap_signout(ctx, victim->name); if (!xprt_found && victim_found) { - xlator_mem_cleanup(victim); - rpcsvc_autoscale_threads(ctx, conf->rpc, -1); + server_call_xlator_mem_cleanup(this, victim->name); } } break; @@ -1613,7 +1711,7 @@ struct volume_options server_options[] = { .type = GF_OPTION_TYPE_TIME, .min = 0, .max = 1013, - .default_value = "42", /* default like network.ping-timeout */ + .default_value = TOSTRING(GF_NETWORK_TIMEOUT), }, { .key = {"transport.*"}, @@ -1628,7 +1726,6 @@ struct volume_options server_options[] = { "in the lru list of the inode cache.", .op_version = {1}, .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC}, - {.key = {"verify-volfile-checksum"}, .type = GF_OPTION_TYPE_BOOL}, {.key = {"trace"}, .type = GF_OPTION_TYPE_BOOL}, { .key = {"config-directory", "conf-dir"}, @@ -1648,13 +1745,22 @@ struct volume_options server_options[] = { "as user bin or group staff.", .op_version = {2}, .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC}, + {.key = {"all-squash"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "off", + .description = "Map requests from any uid/gid to the anonymous " + "uid/gid. Note that this does not apply to any other " + "uids or gids that might be equally sensitive, such " + "as user bin or group staff.", + .op_version = {GD_OP_VERSION_6_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC}, {.key = {"anonuid"}, .type = GF_OPTION_TYPE_INT, .default_value = "65534", /* RPC_NOBODY_UID */ .min = 0, .max = (uint32_t)-1, .description = "value of the uid used for the anonymous " - "user/nfsnobody when root-squash is enabled.", + "user/nfsnobody when root-squash/all-squash is enabled.", .op_version = {3}, .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC}, {.key = {"anongid"}, @@ -1663,7 +1769,7 @@ struct volume_options server_options[] = { .min = 0, .max = (uint32_t)-1, .description = "value of the gid used for the anonymous " - "user/nfsnobody when root-squash is enabled.", + "user/nfsnobody when root-squash/all-squash is enabled.", .op_version = {3}, .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC}, {.key = {"statedump-path"}, @@ -1740,7 +1846,7 @@ struct volume_options server_options[] = { .type = GF_OPTION_TYPE_INT, .min = 1, .max = 1024, - .default_value = "1", + .default_value = "2", .description = "Specifies the number of event threads to execute " "in parallel. Larger values would help process" " responses faster, depending on available processing" |