From 74195df7a5bdc6ae9489d63a3b8157cbb2ddd8e2 Mon Sep 17 00:00:00 2001 From: Anand Avati Date: Wed, 21 Jul 2010 03:53:53 +0000 Subject: mount/fuse: disable fuse support from volfile glusterfsd: implement GETSPEC functionality to fetch volfiles from server Signed-off-by: Anand V. Avati Signed-off-by: Anand V. Avati BUG: 971 (dynamic volume management) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=971 --- glusterfsd/src/Makefile.am | 8 +- glusterfsd/src/fetch-spec.c | 474 ++++++++++++++++++++++---------------------- glusterfsd/src/glusterfsd.c | 151 +++++++------- 3 files changed, 324 insertions(+), 309 deletions(-) (limited to 'glusterfsd') diff --git a/glusterfsd/src/Makefile.am b/glusterfsd/src/Makefile.am index f185f982c..af4ce65fa 100644 --- a/glusterfsd/src/Makefile.am +++ b/glusterfsd/src/Makefile.am @@ -4,13 +4,17 @@ glusterfsd_SOURCES = glusterfsd.c fetch-spec.c if GF_DARWIN_HOST_OS glusterfsd_SOURCES += $(CONTRIBDIR)/apple/daemon.c endif -glusterfsd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD) +glusterfsd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ + $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \ + $(top_builddir)/rpc/xdr/src/libgfxdr.la \ + $(GF_LDADD) glusterfsd_LDFLAGS = $(GF_LDFLAGS) $(GF_GLUSTERFS_LDFLAGS) noinst_HEADERS = glusterfsd.h glusterfsd-common.h glusterfsd-mem-types.h AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\ -I$(top_srcdir)/libglusterfs/src -DDATADIR=\"$(localstatedir)\" \ - -DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS) + -DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS) \ + -I$(top_srcdir)/rpc/rpc-lib/src -I$(top_srcdir)/rpc/xdr/src if GF_DARWIN_HOST_OS AM_CFLAGS += -I$(CONTRIBDIR)/apple endif diff --git a/glusterfsd/src/fetch-spec.c b/glusterfsd/src/fetch-spec.c index f8dcb7584..ac2961062 100644 --- a/glusterfsd/src/fetch-spec.c +++ b/glusterfsd/src/fetch-spec.c @@ -33,271 +33,267 @@ #include "event.h" #include "defaults.h" +#include "rpc-clnt.h" +#include "protocol-common.h" +#include "glusterfs3.h" -#if 0 -int glusterfs_graph_parent_up (xlator_t *graph); +typedef ssize_t (*mgmt_serialize_t) (struct iovec outmsg, void *args); -static int -fetch_cbk (call_frame_t *frame, - void *cookie, - xlator_t *this, - int32_t op_ret, - int32_t op_errno, - char *spec_data) -{ - FILE *spec_fp = NULL; - - spec_fp = frame->local; - - if (op_ret >= 0) { - fwrite (spec_data, strlen (spec_data), 1, spec_fp); - fflush (spec_fp); - fclose (spec_fp); - } - else { - gf_log (frame->this->name, GF_LOG_ERROR, - "GETSPEC from server returned -1 (%s)", - strerror (op_errno)); - } - - frame->local = NULL; - STACK_DESTROY (frame->root); - - /* exit the child process */ - exit (op_ret); -} +char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = { + [GF_HNDSK_NULL] = "NULL", + [GF_HNDSK_SETVOLUME] = "SETVOLUME", + [GF_HNDSK_GETSPEC] = "GETSPEC", + [GF_HNDSK_PING] = "PING", +}; +rpc_clnt_prog_t clnt_handshake_prog = { + .progname = "GlusterFS Handshake", + .prognum = GLUSTER_HNDSK_PROGRAM, + .progver = GLUSTER_HNDSK_VERSION, + .procnames = clnt_handshake_procs, +}; -static int -fetch_notify (xlator_t *this_xl, int event, void *data, ...) + +int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx); +int glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp); + +int +mgmt_submit_request (void *req, call_frame_t *frame, + glusterfs_ctx_t *ctx, + rpc_clnt_prog_t *prog, int procnum, + mgmt_serialize_t sfunc, fop_cbk_fn_t cbkfn) { - int ret = 0; - call_frame_t *frame = NULL; - static int failed_connects = 0; - - switch (event) - { - case GF_EVENT_CHILD_UP: - frame = create_frame (this_xl, this_xl->ctx->pool); - frame->local = this_xl->private; - - STACK_WIND (frame, fetch_cbk, - this_xl->children->xlator, - this_xl->children->xlator->fops->getspec, - this_xl->ctx->cmd_args.volfile_id, - 0); - break; - case GF_EVENT_CHILD_DOWN: - failed_connects++; - if (failed_connects - >= this_xl->ctx->cmd_args.max_connect_attempts) { - exit (1); + int ret = -1; + int count = 0; + struct iovec iov = {0, }; + struct iobuf *iobuf = NULL; + struct iobref *iobref = NULL; + + iobref = iobref_new (); + if (!iobref) { + goto out; + } + + iobuf = iobuf_get (ctx->iobuf_pool); + if (!iobuf) { + goto out; + }; + + iobref_add (iobref, iobuf); + + iov.iov_base = iobuf->ptr; + iov.iov_len = 128 * GF_UNIT_KB; + + + /* Create the xdr payload */ + if (req && sfunc) { + ret = sfunc (iov, req); + if (ret == -1) { + goto out; } - break; - default: - ret = default_notify (this_xl, event, data); - break; - } - - return ret; -} + iov.iov_len = ret; + count = 1; + } + /* Send the msg */ + ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn, + &iov, count, + NULL, 0, iobref, frame); -static int -fetch_init (xlator_t *xl) -{ - return 0; +out: + if (iobref) + iobref_unref (iobref); + + return ret; } -static xlator_t * -get_shrub (glusterfs_ctx_t *ctx, - const char *remote_host, - const char *transport, - uint32_t remote_port) + +/* XXX: move these into @ctx */ +static char oldvolfile[131072]; +static int oldvollen = 0; +static void *timer; + +int +mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count, + void *myframe) { - volume_opt_list_t *vol_opt = NULL; - xlator_t *trav = NULL; - int ret = 0; - xlator_t *top = NULL; - xlator_t *trans = NULL; - xlator_list_t *parent = NULL, *tmp = NULL; - - top = CALLOC (1, sizeof (*top)); - ERR_ABORT (top); - trans = CALLOC (1, sizeof (*trans)); - ERR_ABORT (trans); - - INIT_LIST_HEAD (&top->volume_options); - INIT_LIST_HEAD (&trans->volume_options); - - top->name = "top"; - top->ctx = ctx; - top->next = trans; - top->init = fetch_init; - top->notify = fetch_notify; - top->children = (void *) CALLOC (1, sizeof (*top->children)); - ERR_ABORT (top->children); - top->children->xlator = trans; - - trans->name = "trans"; - trans->ctx = ctx; - trans->prev = top; - trans->init = fetch_init; - trans->notify = default_notify; - trans->options = get_new_dict (); - - parent = CALLOC (1, sizeof(*parent)); - parent->xlator = top; - if (trans->parents == NULL) - trans->parents = parent; - else { - tmp = trans->parents; - while (tmp->next) - tmp = tmp->next; - tmp->next = parent; - } - - /* TODO: log on failure to set dict */ - if (remote_host) { - ret = dict_set (trans->options, "remote-host", - str_to_data ((char *)remote_host)); + gf_getspec_rsp rsp = {0,}; + call_frame_t *frame = NULL; + glusterfs_ctx_t *ctx = NULL; + int ret = 0; + ssize_t size = 0; + FILE *tmpfp = NULL; + struct timeval tv = {0, }; + + frame = myframe; + ctx = frame->this->ctx; + + if (-1 == req->rpc_status) { + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; } - if (remote_port) - ret = dict_set_uint32 (trans->options, "remote-port", - remote_port); - - /* 'option remote-subvolume ' is needed here even though - * its not used - */ - ret = dict_set_static_ptr (trans->options, "remote-subvolume", - "brick"); - ret = dict_set_static_ptr (trans->options, "disable-handshake", "on"); - ret = dict_set_static_ptr (trans->options, "non-blocking-io", "off"); - - if (transport) { - char *transport_type = CALLOC (1, strlen (transport) + 10); - ERR_ABORT (transport_type); - strcpy(transport_type, transport); - - if (strchr (transport_type, ':')) - *(strchr (transport_type, ':')) = '\0'; - - ret = dict_set_dynstr (trans->options, "transport-type", - transport_type); - } - - xlator_set_type (trans, "protocol/client"); - - trav = top; - while (trav) { - /* Get the first volume_option */ - if (!list_empty (&trav->volume_options)) { - list_for_each_entry (vol_opt, - &trav->volume_options, list) - break; - if ((ret = - validate_xlator_volume_options (trav, - vol_opt->given_opt)) < 0) { - gf_log (trav->name, GF_LOG_ERROR, - "validating translator failed"); - return NULL; - } - } - trav = trav->next; - } + ret = xdr_to_getspec_rsp (*iov, &rsp); + if (ret < 0) { + gf_log (frame->this->name, GF_LOG_ERROR, "error"); + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; + } + + if (-1 == rsp.op_ret) { + gf_log (frame->this->name, GF_LOG_ERROR, + "failed to get the 'volume file' from server"); + goto out; + } + + size = rsp.op_ret; + + if (size == oldvollen && (memcmp (oldvolfile, rsp.spec, size) == 0)) + goto out; + + tmpfp = tmpfile (); + if (!tmpfp) + goto out; + + fwrite (rsp.spec, size, 1, tmpfp); + fflush (tmpfp); + + + ret = glusterfs_process_volfp (ctx, tmpfp); + if (ret) + goto out; + + oldvollen = size; + memcpy (oldvolfile, rsp.spec, size); + +out: + tv.tv_sec = 1; + timer = gf_timer_call_after (ctx, tv, + (gf_timer_cbk_t) glusterfs_volfile_fetch, + ctx); - if (xlator_tree_init (top) != 0) - return NULL; + STACK_DESTROY (frame->root); - glusterfs_graph_parent_up (top); - - return top; + if (rsp.spec) + free (rsp.spec); + + return 0; } -static int -_fetch (glusterfs_ctx_t *ctx, - FILE *spec_fp, - const char *remote_host, - const char *transport, - uint32_t remote_port) +int +glusterfs_volfile_fetch (glusterfs_ctx_t *ctx) { - xlator_t *this = NULL; - - this = get_shrub (ctx, remote_host, transport, remote_port); - if (this == NULL) - return -1; - - this->private = spec_fp; - - event_dispatch (ctx->event_pool); - - return 0; + cmd_args_t *cmd_args = NULL; + gf_getspec_req req = {0, }; + int ret = 0; + call_frame_t *frame = NULL; + + + { + if (timer) + gf_timer_call_cancel (ctx, timer); + timer = NULL; + } + + cmd_args = &ctx->cmd_args; + + frame = create_frame (THIS, ctx->pool); + + req.key = cmd_args->volfile_id; + req.flags = 0; + + ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog, + GF_HNDSK_GETSPEC, xdr_from_getspec_req, + mgmt_getspec_cbk); + return ret; } -static int -_fork_and_fetch (glusterfs_ctx_t *ctx, - FILE *spec_fp, - const char *remote_host, - const char *transport, - uint32_t remote_port) +static int +mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, + void *data) { - int ret; - - ret = fork (); - switch (ret) { - case -1: - perror ("fork()"); - break; - case 0: - /* child */ - ret = _fetch (ctx, spec_fp, remote_host, - transport, remote_port); - if (ret == -1) - exit (ret); - default: - /* parent */ - wait (&ret); - ret = WEXITSTATUS (ret); - } - return ret; + xlator_t *this = NULL; + glusterfs_ctx_t *ctx = NULL; + int ret = 0; + + this = mydata; + ctx = this->ctx; + + switch (event) { + case RPC_CLNT_CONNECT: + rpc_clnt_set_connected (ctx->mgmt); + + ret = glusterfs_volfile_fetch (ctx); + + break; + default: + break; + } + + return 0; } -FILE * -fetch_spec (glusterfs_ctx_t *ctx) + +int +glusterfs_mgmt_init (glusterfs_ctx_t *ctx) { - char *remote_host = NULL; - char *transport = NULL; - FILE *spec_fp; - int32_t ret; - - spec_fp = tmpfile (); - - if (!spec_fp) { - perror ("tmpfile ()"); - return NULL; - } - - remote_host = ctx->cmd_args.volfile_server; - transport = ctx->cmd_args.volfile_server_transport; - if (!transport) - transport = "tcp"; - - ret = _fork_and_fetch (ctx, spec_fp, remote_host, transport, - ctx->cmd_args.volfile_server_port); - - if (!ret) { - fseek (spec_fp, 0, SEEK_SET); - } - else { - fclose (spec_fp); - spec_fp = NULL; - } - - return spec_fp; + cmd_args_t *cmd_args = NULL; + struct rpc_clnt *rpc = NULL; + struct rpc_clnt_config rpc_cfg = {0,}; + dict_t *options = NULL; + int ret = -1; + int port = 6969; + char *host = NULL; + + cmd_args = &ctx->cmd_args; + + if (ctx->mgmt) + return 0; + + options = dict_new (); + if (!options) + goto out; + + if (cmd_args->volfile_server_port) + port = cmd_args->volfile_server_port; + + host = "localhost"; + if (cmd_args->volfile_server) + host = cmd_args->volfile_server; + + rpc_cfg.remote_host = host; + rpc_cfg.remote_port = port; + + ret = dict_set_int32 (options, "remote-port", port); + if (ret) + goto out; + + ret = dict_set_str (options, "remote-host", host); + if (ret) + goto out; + + ret = dict_set_str (options, "transport.address-family", "inet"); + if (ret) + goto out; + + rpc = rpc_clnt_init (&rpc_cfg, options, THIS->ctx, THIS->name); + if (!rpc) { + ret = -1; + goto out; + } + + ctx->mgmt = rpc; + + ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS); + if (ret) + goto out; + +out: + return ret; } -#endif diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index dc5536bfb..acd2cb599 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -181,6 +181,7 @@ static struct argp argp = { gf_options, parse_opts, argp_doc, gf_doc }; int glusterfs_pidfile_cleanup (glusterfs_ctx_t *ctx); int glusterfs_volumes_init (glusterfs_ctx_t *ctx); +int glusterfs_mgmt_init (glusterfs_ctx_t *ctx); int create_fuse_mount (glusterfs_ctx_t *ctx) @@ -190,60 +191,52 @@ create_fuse_mount (glusterfs_ctx_t *ctx) xlator_t *master = NULL; cmd_args = &ctx->cmd_args; - master = ctx->master; - if (!master && !cmd_args->mount_point) - return 0; - - if (!master) { - master = GF_CALLOC (1, sizeof (*master), - gfd_mt_xlator_t); - if (!master) - goto err; - master->name = gf_strdup ("fuse"); - if (!master->name) - goto err; + if (!cmd_args->mount_point) + return 0; - if (xlator_set_type (master, "mount/fuse") == -1) { - gf_log ("glusterfsd", GF_LOG_ERROR, - "MOUNT-POINT %s initialization failed", - cmd_args->mount_point); - goto err; - } + master = GF_CALLOC (1, sizeof (*master), + gfd_mt_xlator_t); + if (!master) + goto err; - master->options = get_new_dict (); + master->name = gf_strdup ("fuse"); + if (!master->name) + goto err; - ctx->master = master; + if (xlator_set_type (master, "mount/fuse") == -1) { + gf_log ("glusterfsd", GF_LOG_ERROR, + "MOUNT-POINT %s initialization failed", + cmd_args->mount_point); + goto err; } - master->ctx = ctx; - if (cmd_args->mount_point) - ret = dict_set_static_ptr (master->options, ZR_MOUNTPOINT_OPT, + master->ctx = ctx; + master->options = get_new_dict (); + + ret = dict_set_static_ptr (master->options, ZR_MOUNTPOINT_OPT, cmd_args->mount_point); + if (ret < 0) { + gf_log ("glusterfsd", GF_LOG_ERROR, + "failed to set mount-point to options dictionary"); + goto err; + } - if (ret == 0 && cmd_args->fuse_attribute_timeout >= 0) + if (cmd_args->fuse_attribute_timeout >= 0) ret = dict_set_double (master->options, ZR_ATTR_TIMEOUT_OPT, cmd_args->fuse_attribute_timeout); - - if (ret == 0 && cmd_args->fuse_entry_timeout >= 0) + if (cmd_args->fuse_entry_timeout >= 0) ret = dict_set_double (master->options, ZR_ENTRY_TIMEOUT_OPT, cmd_args->fuse_entry_timeout); - if (ret == 0 && cmd_args->volfile_check) + if (cmd_args->volfile_check) ret = dict_set_int32 (master->options, ZR_STRICT_VOLFILE_CHECK, cmd_args->volfile_check); - if (ret == 0 && cmd_args->dump_fuse) + if (cmd_args->dump_fuse) ret = dict_set_static_ptr (master->options, ZR_DUMP_FUSE, cmd_args->dump_fuse); - if (ret) { - gf_log ("glusterfsd", GF_LOG_ERROR, - "failed to set mount-point to options " - "dictionary"); - goto err; - } - switch (cmd_args->fuse_direct_io_mode) { case GF_OPTION_DISABLE: /* disable */ ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT, @@ -262,6 +255,8 @@ create_fuse_mount (glusterfs_ctx_t *ctx) if (ret) goto err; + ctx->master = master; + return 0; err: @@ -283,23 +278,6 @@ get_volfp (glusterfs_ctx_t *ctx) cmd_args = &ctx->cmd_args; - if (cmd_args->volfile_server) { -// specfp = fetch_spec (ctx); - - if (specfp == NULL) { - gf_log ("glusterfsd", GF_LOG_ERROR, - "error while getting volume file from " - "server %s", cmd_args->volfile_server); - } - else { - gf_log ("glusterfsd", GF_LOG_DEBUG, - "loading volume file from server %s", - cmd_args->volfile_server); - } - - return specfp; - } - ret = sys_lstat (cmd_args->volfile, &statbuf); if (ret == -1) { gf_log ("glusterfsd", GF_LOG_ERROR, @@ -848,7 +826,7 @@ logging_init (glusterfs_ctx_t *ctx) int -parse_cmdline (int argc, char *argv[], cmd_args_t *cmd_args) +parse_cmdline (int argc, char *argv[], glusterfs_ctx_t *ctx) { int process_mode = 0; int ret = 0; @@ -859,6 +837,9 @@ parse_cmdline (int argc, char *argv[], cmd_args_t *cmd_args) char tmp_logfile[1024] = { 0 }; char *tmp_logfile_dyn = NULL; char *tmp_logfilebase = NULL; + cmd_args_t *cmd_args = NULL; + + cmd_args = &ctx->cmd_args; argp_parse (&argp, argc, argv, ARGP_IN_ORDER, NULL, cmd_args); @@ -880,6 +861,12 @@ parse_cmdline (int argc, char *argv[], cmd_args_t *cmd_args) cmd_args->volfile = gf_strdup (DEFAULT_CLIENT_VOLFILE); } + if (cmd_args->volfile_server) { + ret = glusterfs_mgmt_init (ctx); + if (ret) + goto out; + } + if (cmd_args->run_id) { ret = sys_lstat (cmd_args->log_file, &stbuf); /* If its /dev/null, or /dev/stdout, /dev/stderr, @@ -912,7 +899,8 @@ parse_cmdline (int argc, char *argv[], cmd_args_t *cmd_args) } } - return 0; +out: + return ret; } @@ -1162,33 +1150,27 @@ postfork: int -glusterfs_volumes_init (glusterfs_ctx_t *ctx) +glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp) { - FILE *fp = NULL; glusterfs_graph_t *graph = NULL; int ret = 0; - fp = get_volfp (ctx); + graph = glusterfs_graph_construct (fp); - if (!fp) { - gf_log ("glusterfsd", GF_LOG_ERROR, - "Cannot reach volume specification file"); + if (!graph) { ret = -1; goto out; } - graph = glusterfs_graph_construct (fp); + ret = glusterfs_graph_prepare (graph, ctx); - if (!graph) { + if (ret) { + glusterfs_graph_destroy (graph); ret = -1; goto out; } - ret = glusterfs_graph_prepare (graph, ctx); - if (!ret) - ret = create_fuse_mount (ctx); - if (!ret) - ret = glusterfs_graph_activate (graph, ctx); + ret = glusterfs_graph_activate (graph, ctx); if (ret) { glusterfs_graph_destroy (graph); @@ -1199,7 +1181,36 @@ glusterfs_volumes_init (glusterfs_ctx_t *ctx) out: if (fp) fclose (fp); + return ret; +} + + +int +glusterfs_volumes_init (glusterfs_ctx_t *ctx) +{ + FILE *fp = NULL; + cmd_args_t *cmd_args = NULL; + int ret = 0; + cmd_args = &ctx->cmd_args; + + if (cmd_args->volfile_server) + return 0; + + fp = get_volfp (ctx); + + if (!fp) { + gf_log ("glusterfsd", GF_LOG_ERROR, + "Cannot reach volume specification file"); + ret = -1; + goto out; + } + + ret = glusterfs_process_volfp (ctx, fp); + if (ret) + goto out; + +out: return ret; } @@ -1222,7 +1233,7 @@ main (int argc, char *argv[]) if (ret) goto out; - ret = parse_cmdline (argc, argv, &ctx->cmd_args); + ret = parse_cmdline (argc, argv, ctx); if (ret) goto out; @@ -1232,6 +1243,10 @@ main (int argc, char *argv[]) gf_proc_dump_init(); + ret = create_fuse_mount (ctx); + if (ret) + goto out; + ret = daemonize (ctx); if (ret) goto out; -- cgit