diff options
| -rw-r--r-- | configure.ac | 2 | ||||
| -rw-r--r-- | glusterfs.spec.in | 1 | ||||
| -rw-r--r-- | libglusterfs/src/call-stub.h | 7 | ||||
| -rw-r--r-- | libglusterfs/src/glfs-message-id.h | 4 | ||||
| -rw-r--r-- | xlators/experimental/Makefile.am | 2 | ||||
| -rw-r--r-- | xlators/experimental/nsr-client/Makefile.am | 3 | ||||
| -rw-r--r-- | xlators/experimental/nsr-client/src/Makefile.am | 32 | ||||
| -rw-r--r-- | xlators/experimental/nsr-client/src/fop-template.c | 113 | ||||
| -rwxr-xr-x | xlators/experimental/nsr-client/src/gen-fops.py | 57 | ||||
| -rw-r--r-- | xlators/experimental/nsr-client/src/nsr-messages.h | 67 | ||||
| -rw-r--r-- | xlators/experimental/nsr-client/src/nsrc.c | 236 | ||||
| -rw-r--r-- | xlators/experimental/nsr-client/src/nsrc.h | 24 |
12 files changed, 547 insertions, 1 deletions
diff --git a/configure.ac b/configure.ac index 5edff258e0d..7ab49ae44fd 100644 --- a/configure.ac +++ b/configure.ac @@ -187,6 +187,8 @@ AC_CONFIG_FILES([Makefile xlators/mgmt/glusterd/Makefile xlators/mgmt/glusterd/src/Makefile xlators/experimental/Makefile + xlators/experimental/nsr-client/Makefile + xlators/experimental/nsr-client/src/Makefile cli/Makefile cli/src/Makefile doc/Makefile diff --git a/glusterfs.spec.in b/glusterfs.spec.in index 4a8e513b6f7..46a6699ffde 100644 --- a/glusterfs.spec.in +++ b/glusterfs.spec.in @@ -1139,6 +1139,7 @@ fi %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/arbiter.so %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bit-rot.so %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bitrot-stub.so +%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/nsrc.so %if ( 0%{!?_without_tiering:1} ) %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/changetimerecorder.so %endif diff --git a/libglusterfs/src/call-stub.h b/libglusterfs/src/call-stub.h index f34073977e4..01621368ee9 100644 --- a/libglusterfs/src/call-stub.h +++ b/libglusterfs/src/call-stub.h @@ -760,4 +760,11 @@ fop_seek_cbk_stub (call_frame_t *frame, fop_seek_cbk_t fn, void call_resume (call_stub_t *stub); void call_stub_destroy (call_stub_t *stub); void call_unwind_error (call_stub_t *stub, int op_ret, int op_errno); + +/* + * Sometimes we might want to call just this, perhaps repeatedly, without + * having (or being able) to destroy and recreate it. + */ +void call_resume_wind (call_stub_t *stub); + #endif diff --git a/libglusterfs/src/glfs-message-id.h b/libglusterfs/src/glfs-message-id.h index e1b75400bbd..d002fd57fb7 100644 --- a/libglusterfs/src/glfs-message-id.h +++ b/libglusterfs/src/glfs-message-id.h @@ -162,6 +162,10 @@ GLFS_MSGID_COMP_SYMLINK_CACHE_END #define GLFS_MSGID_COMP_SHARD GLFS_MSGID_COMP_CHANGELOG_LIB_END #define GLFS_MSGID_COMP_SHARD_END (GLFS_MSGID_COMP_SHARD +\ GLFS_MSGID_SEGMENT) + +#define GLFS_MSGID_COMP_NSR GLFS_MSGID_COMP_SHARD_END +#define GLFS_MSGID_COMP_NSR_END (GLFS_MSGID_COMP_SHARD_END+\ + GLFS_MSGID_SEGMENT) /* --- new segments for messages goes above this line --- */ #endif /* !_GLFS_MESSAGE_ID_H_ */ diff --git a/xlators/experimental/Makefile.am b/xlators/experimental/Makefile.am index e182a87eb77..f937146c72a 100644 --- a/xlators/experimental/Makefile.am +++ b/xlators/experimental/Makefile.am @@ -1,3 +1,3 @@ -SUBDIRS = +SUBDIRS = nsr-client CLEANFILES = diff --git a/xlators/experimental/nsr-client/Makefile.am b/xlators/experimental/nsr-client/Makefile.am new file mode 100644 index 00000000000..a985f42a877 --- /dev/null +++ b/xlators/experimental/nsr-client/Makefile.am @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = diff --git a/xlators/experimental/nsr-client/src/Makefile.am b/xlators/experimental/nsr-client/src/Makefile.am new file mode 100644 index 00000000000..0dbba285cc9 --- /dev/null +++ b/xlators/experimental/nsr-client/src/Makefile.am @@ -0,0 +1,32 @@ +xlator_LTLIBRARIES = nsrc.la +xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental + +nodist_nsrc_la_SOURCES = nsrc-cg.c +CLEANFILES = $(nodist_nsrc_la_SOURCES) + +nsrc_la_LDFLAGS = -module -avoid-version +nsrc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la + +noinst_HEADERS = \ + $(top_srcdir)/xlators/lib/src/libxlator.h \ + $(top_srcdir)/glusterfsd/src/glusterfsd.h \ + nsrc.h nsr-messages.h + +AM_CPPFLAGS = $(GF_CPPFLAGS) \ + -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \ + -I$(top_srcdir)/rpc/rpc-lib/src + +AM_CFLAGS = -Wall $(GF_CFLAGS) + +NSRC_PREFIX = $(top_srcdir)/xlators/experimental/nsr-client/src +NSRC_GEN_FOPS = $(NSRC_PREFIX)/gen-fops.py +NSRC_TEMPLATES = $(NSRC_PREFIX)/fop-template.c +NSRC_WRAPPER = $(NSRC_PREFIX)/nsrc.c +noinst_PYTHON = $(NSRC_GEN_FOPS) +EXTRA_DIST = $(NSRC_TEMPLATES) $(NSRC_WRAPPER) + +nsrc-cg.c: $(NSRC_GEN_FOPS) $(NSRC_TEMPLATES) $(NSRC_WRAPPER) + $(PYTHON) $(NSRC_GEN_FOPS) $(NSRC_TEMPLATES) $(NSRC_WRAPPER) > $@ + +uninstall-local: + rm -f $(DESTDIR)$(xlatordir)/nsr.so diff --git a/xlators/experimental/nsr-client/src/fop-template.c b/xlators/experimental/nsr-client/src/fop-template.c new file mode 100644 index 00000000000..59708732aa0 --- /dev/null +++ b/xlators/experimental/nsr-client/src/fop-template.c @@ -0,0 +1,113 @@ +/* template-name fop */ +int32_t +nsrc_@NAME@ (call_frame_t *frame, xlator_t *this, + @LONG_ARGS@) +{ + nsrc_local_t *local = NULL; + xlator_t *target_xl = ACTIVE_CHILD(this); + + local = mem_get(this->local_pool); + if (!local) { + goto err; + } + + local->stub = fop_@NAME@_stub (frame, nsrc_@NAME@_continue, + @SHORT_ARGS@); + if (!local->stub) { + goto err; + } + local->curr_xl = target_xl; + local->scars = 0; + + frame->local = local; + STACK_WIND_COOKIE (frame, nsrc_@NAME@_cbk, target_xl, + target_xl, target_xl->fops->@NAME@, + @SHORT_ARGS@); + return 0; + +err: + if (local) { + mem_put(local); + } + STACK_UNWIND_STRICT (@NAME@, frame, -1, ENOMEM, + @ERROR_ARGS@); + return 0; +} + +/* template-name cbk */ +int32_t +nsrc_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, + @LONG_ARGS@) +{ + nsrc_local_t *local = frame->local; + xlator_t *last_xl = cookie; + xlator_t *next_xl; + nsrc_private_t *priv = this->private; + struct timespec spec; + + if (op_ret != (-1)) { + if (local->scars) { + gf_msg (this->name, GF_LOG_INFO, 0, N_MSG_RETRY_MSG, + HILITE("retried %p OK"), frame->local); + } + priv->active = last_xl; + goto unwind; + } + if ((op_errno != EREMOTE) && (op_errno != ENOTCONN)) { + goto unwind; + } + + /* TBD: get leader ID from xdata? */ + next_xl = next_xlator(this, last_xl); + /* + * We can't just give up after we've tried all bricks, because it's + * quite likely that a new leader election just hasn't finished yet. + * We also shouldn't retry endlessly, and especially not at a high + * rate, but that's good enough while we work on other things. + * + * TBD: implement slow/finite retry via a worker thread + */ + if (!next_xl || (local->scars >= SCAR_LIMIT)) { + gf_msg (this->name, GF_LOG_DEBUG, 0, N_MSG_RETRY_MSG, + HILITE("ran out of retries for %p"), frame->local); + goto unwind; + } + + local->curr_xl = next_xl; + local->scars += 1; + spec.tv_sec = 1; + spec.tv_nsec = 0; + /* + * WARNING + * + * Just calling gf_timer_call_after like this leaves open the + * possibility that writes will get reordered, if a first write is + * rescheduled and then a second comes along to find an updated + * priv->active before the first actually executes. We might need to + * implement a stricter (and more complicated) queuing mechanism to + * ensure absolute consistency in this case. + */ + if (gf_timer_call_after(this->ctx, spec, nsrc_retry_cb, local)) { + return 0; + } + +unwind: + call_stub_destroy(local->stub); + STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno, + @SHORT_ARGS@); + return 0; +} + +/* template-name cont-func */ +int32_t +nsrc_@NAME@_continue (call_frame_t *frame, xlator_t *this, + @LONG_ARGS@) +{ + nsrc_local_t *local = frame->local; + + STACK_WIND_COOKIE (frame, nsrc_@NAME@_cbk, local->curr_xl, + local->curr_xl, local->curr_xl->fops->@NAME@, + @SHORT_ARGS@); + return 0; +} diff --git a/xlators/experimental/nsr-client/src/gen-fops.py b/xlators/experimental/nsr-client/src/gen-fops.py new file mode 100755 index 00000000000..4d9451f7177 --- /dev/null +++ b/xlators/experimental/nsr-client/src/gen-fops.py @@ -0,0 +1,57 @@ +#!/usr/bin/python + +import os +import re +import string +import sys + +curdir = os.path.dirname(sys.argv[0]) +gendir = os.path.join(curdir,'../../../../libglusterfs/src') +sys.path.append(gendir) +from generator import ops, fop_subs, cbk_subs, generate + +# We really want the callback argument list, even when we're generating fop +# code, so we propagate here. +# TBD: this should probably be right in generate.py +for k, v in cbk_subs.iteritems(): + fop_subs[k]['@ERROR_ARGS@'] = v['@ERROR_ARGS@'] + +# Stolen from old codegen.py +def load_templates (path): + templates = {} + tmpl_re = re.compile("/\* template-name (.*) \*/") + templates = {} + t_name = None + for line in open(path,"r").readlines(): + if not line: + break + m = tmpl_re.match(line) + if m: + if t_name: + templates[t_name] = string.join(t_contents,'') + t_name = m.group(1).strip() + t_contents = [] + elif t_name: + t_contents.append(line) + if t_name: + templates[t_name] = string.join(t_contents,'') + return templates + +# Stolen from gen_fdl.py +def gen_client (templates): + for name, value in ops.iteritems(): + if name == 'getspec': + # It's not real if it doesn't have a stub function. + continue + print generate(templates['cbk'],name,cbk_subs) + print generate(templates['cont-func'],name,fop_subs) + print generate(templates['fop'],name,fop_subs) + +tmpl = load_templates(sys.argv[1]) +for l in open(sys.argv[2],'r').readlines(): + if l.find('#pragma generate') != -1: + print "/* BEGIN GENERATED CODE - DO NOT MODIFY */" + gen_client(tmpl) + print "/* END GENERATED CODE */" + else: + print l[:-1] diff --git a/xlators/experimental/nsr-client/src/nsr-messages.h b/xlators/experimental/nsr-client/src/nsr-messages.h new file mode 100644 index 00000000000..d68cfc5d228 --- /dev/null +++ b/xlators/experimental/nsr-client/src/nsr-messages.h @@ -0,0 +1,67 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _NSR_MESSAGES_H_ +#define _NSR_MESSAGES_H_ + +#include "glfs-message-id.h" + +/* NOTE: Rules for message additions + * 1) Each instance of a message is _better_ left with a unique message ID, even + * if the message format is the same. Reasoning is that, if the message + * format needs to change in one instance, the other instances are not + * impacted or the new change does not change the ID of the instance being + * modified. + * 2) Addition of a message, + * - Should increment the GLFS_NUM_MESSAGES + * - Append to the list of messages defined, towards the end + * - Retain macro naming as glfs_msg_X (for redability across developers) + * NOTE: Rules for message format modifications + * 3) Check acorss the code if the message ID macro in question is reused + * anywhere. If reused then then the modifications should ensure correctness + * everywhere, or needs a new message ID as (1) above was not adhered to. If + * not used anywhere, proceed with the required modification. + * NOTE: Rules for message deletion + * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used + * anywhere, then can be deleted, but will leave a hole by design, as + * addition rules specify modification to the end of the list and not filling + * holes. + */ + +#define NSR_COMP_BASE GLFS_MSGID_COMP_NSR +#define GLFS_NUM_MESSAGES 1 +#define GLFS_MSGID_END (NSR_COMP_BASE + GLFS_NUM_MESSAGES + 1) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define N_MSG_INIT_FAIL (NSR_COMP_BASE + 1) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define N_MSG_RETRY_MSG (NSR_COMP_BASE + 2) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define N_MSG_MEM_ERR (NSR_COMP_BASE + 3) + + + + + +#endif /* _NSR_MESSAGES_H_ */ diff --git a/xlators/experimental/nsr-client/src/nsrc.c b/xlators/experimental/nsr-client/src/nsrc.c new file mode 100644 index 00000000000..dd3ad20544e --- /dev/null +++ b/xlators/experimental/nsr-client/src/nsrc.c @@ -0,0 +1,236 @@ +/* + Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com> + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "call-stub.h" +#include "defaults.h" +#include "timer.h" +#include "xlator.h" +#include "nsr-messages.h" +#include "nsrc.h" + +#define SCAR_LIMIT 20 +#define HILITE(x) ("[1;33m"x"[0m") + +/* + * The fops are actually generated by gen-fops.py; the rest was mostly copied + * from defaults.c (commit cd253754 on 27 August 2013). + */ + +enum gf_dht_mem_types_ { + gf_mt_nsrc_private_t = gf_common_mt_end + 1, + gf_mt_nsrc_end +}; + +char *NSRC_XATTR = "user.nsr.active"; + +static inline +xlator_t * +ACTIVE_CHILD (xlator_t *parent) +{ + nsrc_private_t *priv = parent->private; + + return priv ? priv->active : FIRST_CHILD(parent); +} + +xlator_t * +next_xlator (xlator_t *this, xlator_t *prev) +{ + xlator_list_t *trav; + + for (trav = this->children; trav; trav = trav->next) { + if (trav->xlator == prev) { + return trav->next ? trav->next->xlator + : this->children->xlator; + } + } + + return NULL; +} + +void +nsrc_retry_cb (void *cb_arg) +{ + nsrc_local_t *local = cb_arg; + + gf_msg (__func__, GF_LOG_INFO, 0, N_MSG_RETRY_MSG, + HILITE("retrying %p"), local); + call_resume_wind(local->stub); +} + +#pragma generate + +int32_t +nsrc_forget (xlator_t *this, inode_t *inode) +{ + gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, N_MSG_INIT_FAIL, + "xlator does not implement forget_cbk"); + return 0; +} + + +int32_t +nsrc_releasedir (xlator_t *this, fd_t *fd) +{ + gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, N_MSG_INIT_FAIL, + "xlator does not implement releasedir_cbk"); + return 0; +} + +int32_t +nsrc_release (xlator_t *this, fd_t *fd) +{ + gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, N_MSG_INIT_FAIL, + "xlator does not implement release_cbk"); + return 0; +} + +struct xlator_fops fops = { + .lookup = nsrc_lookup, + .stat = nsrc_stat, + .fstat = nsrc_fstat, + .truncate = nsrc_truncate, + .ftruncate = nsrc_ftruncate, + .access = nsrc_access, + .readlink = nsrc_readlink, + .mknod = nsrc_mknod, + .mkdir = nsrc_mkdir, + .unlink = nsrc_unlink, + .rmdir = nsrc_rmdir, + .symlink = nsrc_symlink, + .rename = nsrc_rename, + .link = nsrc_link, + .create = nsrc_create, + .open = nsrc_open, + .readv = nsrc_readv, + .writev = nsrc_writev, + .flush = nsrc_flush, + .fsync = nsrc_fsync, + .opendir = nsrc_opendir, + .readdir = nsrc_readdir, + .readdirp = nsrc_readdirp, + .fsyncdir = nsrc_fsyncdir, + .statfs = nsrc_statfs, + .setxattr = nsrc_setxattr, + .getxattr = nsrc_getxattr, + .fsetxattr = nsrc_fsetxattr, + .fgetxattr = nsrc_fgetxattr, + .removexattr = nsrc_removexattr, + .fremovexattr = nsrc_fremovexattr, + .lk = nsrc_lk, + .inodelk = nsrc_inodelk, + .finodelk = nsrc_finodelk, + .entrylk = nsrc_entrylk, + .fentrylk = nsrc_fentrylk, + .rchecksum = nsrc_rchecksum, + .xattrop = nsrc_xattrop, + .fxattrop = nsrc_fxattrop, + .setattr = nsrc_setattr, + .fsetattr = nsrc_fsetattr, + .fallocate = nsrc_fallocate, + .discard = nsrc_discard, +}; + +struct xlator_cbks cbks = { +}; + + +int32_t +mem_acct_init (xlator_t *this) +{ + int ret = -1; + + GF_VALIDATE_OR_GOTO ("nsrc", this, out); + + ret = xlator_mem_acct_init (this, gf_mt_nsrc_end + 1); + + if (ret != 0) { + gf_msg (this->name, GF_LOG_ERROR, ENOMEM, N_MSG_MEM_ERR, + "Memory accounting init failed"); + return ret; + } +out: + return ret; +} + + +int32_t +nsrc_init (xlator_t *this) +{ + nsrc_private_t *priv = NULL; + + this->local_pool = mem_pool_new (nsrc_local_t, 128); + if (!this->local_pool) { + gf_msg (this->name, GF_LOG_ERROR, ENOMEM, N_MSG_MEM_ERR, + "failed to create nsrc_local_t pool"); + goto err; + } + + priv = GF_CALLOC (1, sizeof (*priv), gf_mt_nsrc_private_t); + if (!priv) { + goto err; + } + + priv->active = FIRST_CHILD(this); + this->private = priv; + return 0; + +err: + if (priv) { + GF_FREE(priv); + } + return -1; +} + +void +nsrc_fini (xlator_t *this) +{ + GF_FREE(this->private); +} + +int32_t +nsrc_notify (xlator_t *this, int32_t event, void *data, ...) +{ + int32_t ret = 0; + + switch (event) { + case GF_EVENT_CHILD_DOWN: + /* + * TBD: handle this properly + * + * What we really should do is propagate this only if it caused + * us to lose quorum, and likewise for GF_EVENT_CHILD_UP only + * if it caused us to gain quorum. However, that requires + * tracking child states and for now it's easier to swallow + * these unconditionally. The consequence of failing to do + * this is that DHT sees the first GF_EVENT_CHILD_DOWN and gets + * confused, so it doesn't call us and doesn't get up-to-date + * directory listings etc. + */ + break; + default: + ret = default_notify (this, event, data); + } + + return ret; +} + +class_methods_t class_methods = { + .init = nsrc_init, + .fini = nsrc_fini, + .notify = nsrc_notify, +}; + +struct volume_options options[] = { + { .key = {NULL} }, +}; diff --git a/xlators/experimental/nsr-client/src/nsrc.h b/xlators/experimental/nsr-client/src/nsrc.h new file mode 100644 index 00000000000..0c61d7a9fa8 --- /dev/null +++ b/xlators/experimental/nsr-client/src/nsrc.h @@ -0,0 +1,24 @@ +/* + Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _NSRC_H_ +#define _NSRC_H_ + +typedef struct { + xlator_t *active; +} nsrc_private_t; + +typedef struct { + call_stub_t *stub; + xlator_t *curr_xl; + uint16_t scars; +} nsrc_local_t; + +#endif /* _NSRC_H_ */ |