diff options
| -rw-r--r-- | glusterfsd/src/Makefile.am | 2 | ||||
| -rw-r--r-- | glusterfsd/src/glusterfsd-mgmt.c (renamed from glusterfsd/src/fetch-spec.c) | 156 | ||||
| -rw-r--r-- | glusterfsd/src/glusterfsd.c | 25 | ||||
| -rw-r--r-- | glusterfsd/src/glusterfsd.h | 5 | ||||
| -rw-r--r-- | libglusterfs/src/glusterfs.h | 4 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-handshake.c | 41 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-pmap.c | 52 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-pmap.h | 1 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-utils.c | 14 |
9 files changed, 282 insertions, 18 deletions
diff --git a/glusterfsd/src/Makefile.am b/glusterfsd/src/Makefile.am index af4ce65fa..8cfe4b554 100644 --- a/glusterfsd/src/Makefile.am +++ b/glusterfsd/src/Makefile.am @@ -1,6 +1,6 @@ sbin_PROGRAMS = glusterfsd -glusterfsd_SOURCES = glusterfsd.c fetch-spec.c +glusterfsd_SOURCES = glusterfsd.c glusterfsd-mgmt.c if GF_DARWIN_HOST_OS glusterfsd_SOURCES += $(CONTRIBDIR)/apple/daemon.c endif diff --git a/glusterfsd/src/fetch-spec.c b/glusterfsd/src/glusterfsd-mgmt.c index d9e31b3ec..e6cfa6ac6 100644 --- a/glusterfsd/src/fetch-spec.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -36,10 +36,30 @@ #include "rpc-clnt.h" #include "protocol-common.h" #include "glusterfs3.h" +#include "portmap.h" +static char is_mgmt_rpc_reconnect; typedef ssize_t (*mgmt_serialize_t) (struct iovec outmsg, void *args); + +char *clnt_pmap_procs[GF_PMAP_MAXVALUE] = { + [GF_PMAP_NULL] = "NULL", + [GF_PMAP_PORTBYBRICK] = "PORTBYBRICK", + [GF_PMAP_BRICKBYPORT] = "BRICKBYPORT", + [GF_PMAP_SIGNIN] = "SIGNIN", + [GF_PMAP_SIGNOUT] = "SIGNOUT", + [GF_PMAP_SIGNUP] = "SIGNUP", +}; + + +rpc_clnt_prog_t clnt_pmap_prog = { + .progname = "Gluster Portmap", + .prognum = GLUSTER_PMAP_PROGRAM, + .progver = GLUSTER_PMAP_VERSION, + .procnames = clnt_pmap_procs, +}; + char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = { [GF_HNDSK_NULL] = "NULL", [GF_HNDSK_SETVOLUME] = "SETVOLUME", @@ -55,6 +75,7 @@ rpc_clnt_prog_t clnt_handshake_prog = { }; +int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx); int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx); int glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp); @@ -161,13 +182,16 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count, fwrite (rsp.spec, size, 1, tmpfp); fflush (tmpfp); - ret = glusterfs_process_volfp (ctx, tmpfp); if (ret) goto out; oldvollen = size; memcpy (oldvolfile, rsp.spec, size); + if (!is_mgmt_rpc_reconnect) { + glusterfs_mgmt_pmap_signin (ctx); + is_mgmt_rpc_reconnect = 1; + } out: tv.tv_sec = 1; @@ -192,7 +216,6 @@ glusterfs_volfile_fetch (glusterfs_ctx_t *ctx) int ret = 0; call_frame_t *frame = NULL; - { if (timer) gf_timer_call_cancel (ctx, timer); @@ -230,6 +253,8 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, ret = glusterfs_volfile_fetch (ctx); + if (is_mgmt_rpc_reconnect) + glusterfs_mgmt_pmap_signin (ctx); break; default: break; @@ -297,3 +322,130 @@ out: return ret; } + +static int +mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count, + void *myframe) +{ + pmap_signin_rsp rsp = {0,}; + call_frame_t *frame = NULL; + int ret = 0; + + frame = myframe; + + if (-1 == req->rpc_status) { + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; + } + + ret = xdr_to_pmap_signin_rsp (*iov, &rsp); + if (ret < 0) { + gf_log (frame->this->name, GF_LOG_ERROR, "error"); + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; + } + + if (-1 == rsp.op_ret) { + gf_log (frame->this->name, GF_LOG_ERROR, + "failed to register the port with glusterd"); + goto out; + } +out: + + STACK_DESTROY (frame->root); + return 0; +} + +int +glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx) +{ + call_frame_t *frame = NULL; + pmap_signin_req req = {0, }; + int ret = -1; + cmd_args_t *cmd_args = NULL; + + frame = create_frame (THIS, ctx->pool); + cmd_args = &ctx->cmd_args; + + if (!cmd_args->brick_port || !cmd_args->brick_name) { + gf_log ("fsd-mgmt", GF_LOG_DEBUG, + "portmapper signin arguments not given"); + goto out; + } + + req.port = cmd_args->brick_port; + req.brick = cmd_args->brick_name; + + ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog, + GF_PMAP_SIGNIN, xdr_from_pmap_signin_req, + mgmt_pmap_signin_cbk); + +out: + return ret; +} + + +static int +mgmt_pmap_signout_cbk (struct rpc_req *req, struct iovec *iov, int count, + void *myframe) +{ + pmap_signout_rsp rsp = {0,}; + call_frame_t *frame = NULL; + int ret = 0; + + frame = myframe; + + if (-1 == req->rpc_status) { + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; + } + + ret = xdr_to_pmap_signout_rsp (*iov, &rsp); + if (ret < 0) { + gf_log (frame->this->name, GF_LOG_ERROR, "error"); + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; + } + + if (-1 == rsp.op_ret) { + gf_log (frame->this->name, GF_LOG_ERROR, + "failed to register the port with glusterd"); + goto out; + } +out: + if (frame) + STACK_DESTROY (frame->root); + return 0; +} + + +int +glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx) +{ + int ret = 0; + pmap_signout_req req = {0, }; + call_frame_t *frame = NULL; + cmd_args_t *cmd_args = NULL; + + frame = create_frame (THIS, ctx->pool); + cmd_args = &ctx->cmd_args; + + if (!cmd_args->brick_port || !cmd_args->brick_name) { + gf_log ("fsd-mgmt", GF_LOG_DEBUG, + "portmapper signout arguments not given"); + goto out; + } + + req.port = cmd_args->brick_port; + req.brick = cmd_args->brick_name; + + ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog, + GF_PMAP_SIGNOUT, xdr_from_pmap_signout_req, + mgmt_pmap_signout_cbk); +out: + return ret; +} diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index 6daeac129..c944075f7 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -157,7 +157,11 @@ static struct argp_option gf_options[] = { #else "[default: \"off\"]" #endif - }, + }, + {"brick-name", ARGP_BRICK_NAME_KEY, "BRICK-NAME", OPTION_HIDDEN, + "Brick name to be registered with Gluster portmapper" }, + {"brick-port", ARGP_BRICK_PORT_KEY, "BRICK-PORT", OPTION_HIDDEN, + "Brick Port to be registered with Gluster portmapper" }, {0, 0, 0, 0, "Fuse options:"}, {"direct-io-mode", ARGP_DIRECT_IO_MODE_KEY, "BOOL", OPTION_ARG_OPTIONAL, @@ -574,6 +578,20 @@ parse_opts (int key, char *arg, struct argp_state *state) case ARGP_DUMP_FUSE_KEY: cmd_args->dump_fuse = gf_strdup (arg); break; + case ARGP_BRICK_NAME_KEY: + cmd_args->brick_name = gf_strdup (arg); + break; + case ARGP_BRICK_PORT_KEY: + n = 0; + + if (gf_string2uint_base10 (arg, &n) == 0) { + cmd_args->brick_port = n; + break; + } + + argp_failure (state, -1, 0, + "unknown brick (listen) port %s", arg); + break; } return 0; @@ -588,13 +606,16 @@ cleanup_and_exit (int signum) ctx = glusterfs_ctx_get (); + /* TODO: is this the right place? */ + glusterfs_mgmt_pmap_signout (ctx); + gf_log ("glusterfsd", GF_LOG_NORMAL, "shutting down"); tmp_pool = ctx->pool; mem_pool_destroy (tmp_pool->frame_mem_pool); mem_pool_destroy (tmp_pool->stack_mem_pool); tmp_pool = NULL; - mem_pool_destroy (ctx->stub_mem_pool); + mem_pool_destroy (ctx->stub_mem_pool); glusterfs_pidfile_cleanup (ctx); diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h index 7827c9800..a082b0e4b 100644 --- a/glusterfsd/src/glusterfsd.h +++ b/glusterfsd/src/glusterfsd.h @@ -73,10 +73,15 @@ enum argp_option_keys { ARGP_READ_ONLY_KEY = 148, ARGP_MAC_COMPAT_KEY = 149, ARGP_DUMP_FUSE_KEY = 150, + ARGP_BRICK_NAME_KEY = 151, + ARGP_BRICK_PORT_KEY = 152, }; /* Moved here from fetch-spec.h */ FILE *fetch_spec (glusterfs_ctx_t *ctx); +int glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx); +int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx); + #endif /* __GLUSTERFSD_H__ */ diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h index 6f40b9880..4b157430b 100644 --- a/libglusterfs/src/glusterfs.h +++ b/libglusterfs/src/glusterfs.h @@ -241,6 +241,10 @@ struct _cmd_args { /* key args */ char *mount_point; char *volfile_id; + + /* required for portmap */ + int brick_port; + char *brick_name; }; typedef struct _cmd_args cmd_args_t; diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c index 9f5e03f53..43a937317 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handshake.c +++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c @@ -42,13 +42,44 @@ static size_t build_volfile_path (const char *volname, char *path, size_t path_len) { - int32_t ret = -1; - glusterd_conf_t *priv = NULL; + struct stat stbuf = {0,}; + int32_t ret = -1; + glusterd_conf_t *priv = NULL; + char *vol = NULL; + char *dup_volname = NULL; + char *tmp = NULL; + glusterd_volinfo_t *volinfo = NULL; priv = THIS->private; + dup_volname = gf_strdup (volname); + + ret = glusterd_volinfo_find (dup_volname, &volinfo); + if (ret) { + /* Split the volume name */ + vol = strtok_r (dup_volname, "-", &tmp); + if (!vol) + goto out; + vol = strtok_r (NULL, "-", &tmp); + if (!vol) + goto out; + ret = glusterd_volinfo_find (vol, &volinfo); + if (ret) + goto out; + } + ret = snprintf (path, path_len, "%s/vols/%s/%s.vol", + priv->workdir, volinfo->volname, volname); + if (ret == -1) + goto out; + + ret = stat (path, &stbuf); + if ((ret == -1) && (errno == ENOENT)) + ret = snprintf (path, path_len, "%s/vols/%s/%s-tcp.vol", + priv->workdir, volinfo->volname, volname); - ret = snprintf (path, path_len, "%s/vols/%s/%s-tcp.vol", - priv->workdir, volname, volname); + ret = 1; +out: + if (dup_volname) + GF_FREE (dup_volname); return ret; } @@ -139,6 +170,8 @@ fail: if (cookie) rsp.op_errno = cookie; + if (!rsp.spec) + rsp.spec = ""; glusterd_submit_reply (req, &rsp, NULL, 0, NULL, (gd_serialize_t)xdr_serialize_getspec_rsp); diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c index 3ae513393..3a45da000 100644 --- a/xlators/mgmt/glusterd/src/glusterd-pmap.c +++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c @@ -125,6 +125,25 @@ pmap_registry_search (xlator_t *this, const char *brickname) } +char * +pmap_registry_search_by_port (xlator_t *this, int port) +{ + struct pmap_registry *pmap = NULL; + char *brickname = NULL; + + if (port > 65535) + goto out; + + pmap = pmap_registry_get (this); + + if (pmap->ports[port].used) + brickname = pmap->ports[port].brickname; + +out: + return brickname; +} + + int pmap_registry_alloc (xlator_t *this) { @@ -171,6 +190,26 @@ out: return 0; } +int +pmap_registry_remove (xlator_t *this, int port, const char *brickname) +{ + struct pmap_registry *pmap = NULL; + int p = 0; + + pmap = pmap_registry_get (this); + + if (port > 65535) + goto out; + + p = port; + pmap->ports[p].used = 0; + if (pmap->ports[p].brickname) + free (pmap->ports[p].brickname); + +out: + return 0; +} + typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *data); @@ -200,7 +239,6 @@ gluster_pmap_portbybrick (rpcsvc_request_t *req) char *brick = NULL; int port = 0; - if (xdr_to_glusterfs_req (req, &args, xdr_to_pmap_port_by_brick_req)) { req->rpc_err = GARBAGE_ARGS; goto fail; @@ -229,13 +267,16 @@ gluster_pmap_brickbyport (rpcsvc_request_t *req) pmap_brick_by_port_req args = {0,}; pmap_brick_by_port_rsp rsp = {0,}; - if (xdr_to_glusterfs_req (req, &args, xdr_to_pmap_brick_by_port_req)) { req->rpc_err = GARBAGE_ARGS; goto fail; } - + rsp.brick = pmap_registry_search_by_port (THIS, args.port); + if (!rsp.brick) { + rsp.op_ret = -1; + rsp.brick = ""; + } fail: glusterd_submit_reply (req, &rsp, NULL, 0, NULL, @@ -258,6 +299,7 @@ gluster_pmap_signup (rpcsvc_request_t *req) goto fail; } + rsp.op_ret = pmap_registry_bind (THIS, args.port, args.brick); fail: glusterd_submit_reply (req, &rsp, NULL, 0, NULL, @@ -273,13 +315,14 @@ gluster_pmap_signin (rpcsvc_request_t *req) pmap_signin_req args = {0,}; pmap_signin_rsp rsp = {0,}; - if (xdr_to_glusterfs_req (req, &args, xdr_to_pmap_signin_req)) { req->rpc_err = GARBAGE_ARGS; goto fail; } + rsp.op_ret = pmap_registry_bind (THIS, args.port, args.brick); + fail: glusterd_submit_reply (req, &rsp, NULL, 0, NULL, (gd_serialize_t)xdr_from_pmap_signin_rsp); @@ -302,6 +345,7 @@ gluster_pmap_signout (rpcsvc_request_t *req) goto fail; } + rsp.op_ret = pmap_registry_remove (THIS, args.port, args.brick); fail: glusterd_submit_reply (req, &rsp, NULL, 0, NULL, diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h index dd72012a7..4bc4536f8 100644 --- a/xlators/mgmt/glusterd/src/glusterd-pmap.h +++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h @@ -51,5 +51,6 @@ struct pmap_registry { int pmap_registry_alloc (xlator_t *this); int pmap_registry_bind (xlator_t *this, int port, const char *brickname); +int pmap_registry_remove (xlator_t *this, int port, const char *brickname); #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index beb418583..4202f2716 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -763,16 +763,18 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, port = pmap_registry_alloc (THIS); GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname, count); - snprintf (volfile, PATH_MAX, "%s/%s-%s-%d.vol", path, - brickinfo->hostname, volinfo->volname, count); + snprintf (volfile, PATH_MAX, "%s-%s-%d", brickinfo->hostname, + volinfo->volname, count); snprintf (cmd_str, 8192, - "glusterfs --xlator-option server-*.listen-port=%d -f %s -p %s", - port, volfile, pidfile); + "glusterfs --xlator-option server-*.listen-port=%d " + "-s localhost --volfile-id %s -p %s --brick-name %s " + "--brick-port %d", + port, volfile, pidfile, brickinfo->path, port); ret = system (cmd_str); if (ret == 0) { - pmap_registry_bind (THIS, port, brickinfo->path); + //pmap_registry_bind (THIS, port, brickinfo->path); brickinfo->port = port; } out: @@ -832,6 +834,8 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t *volinfo, goto out; } + //pmap_registry_remove (THIS, brickinfo->port, brickinfo->path); + ret = unlink (pidfile); if (ret) { |