NSR: nsr client code generation patch

This version of the client checks if the error returned
is EREMOTE or ENOTCONN, and if not unwnds the error back

In case of a EREMOTE or ENOTCONN error, it retries on
all the bricks in the replica subgroup, and if the error
still persists, it waits for a sec before going through the
same exercise again.

Change-Id: I916bed32f0820f381dd60fdde3d05b71c69a34dc
Signed-off-by: Avra Sengupta <asengupt@redhat.com>
Reviewed-on: http://review.gluster.org/12388
Smoke: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>

12 files changed, 547 insertions, 1 deletions

diff --git a/configure.ac b/configure.ac
index 5edff258e0d..7ab49ae44fd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -187,6 +187,8 @@ AC_CONFIG_FILES([Makefile
                 xlators/mgmt/glusterd/Makefile
                 xlators/mgmt/glusterd/src/Makefile
                 xlators/experimental/Makefile
+                xlators/experimental/nsr-client/Makefile
+                xlators/experimental/nsr-client/src/Makefile
                 cli/Makefile
                 cli/src/Makefile
                 doc/Makefile
diff --git a/glusterfs.spec.in b/glusterfs.spec.in
index 4a8e513b6f7..46a6699ffde 100644
--- a/glusterfs.spec.in
+++ b/glusterfs.spec.in
@@ -1139,6 +1139,7 @@ fi
 %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/arbiter.so
 %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bit-rot.so
 %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bitrot-stub.so
+%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/nsrc.so
 %if ( 0%{!?_without_tiering:1} )
 %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/changetimerecorder.so
 %endif
diff --git a/libglusterfs/src/call-stub.h b/libglusterfs/src/call-stub.h
index f34073977e4..01621368ee9 100644
--- a/libglusterfs/src/call-stub.h
+++ b/libglusterfs/src/call-stub.h
@@ -760,4 +760,11 @@ fop_seek_cbk_stub (call_frame_t *frame, fop_seek_cbk_t fn,
 void call_resume (call_stub_t *stub);
 void call_stub_destroy (call_stub_t *stub);
 void call_unwind_error (call_stub_t *stub, int op_ret, int op_errno);
+
+/*
+ * Sometimes we might want to call just this, perhaps repeatedly, without
+ * having (or being able) to destroy and recreate it.
+ */
+void call_resume_wind (call_stub_t *stub);
+
 #endif
diff --git a/libglusterfs/src/glfs-message-id.h b/libglusterfs/src/glfs-message-id.h
index e1b75400bbd..d002fd57fb7 100644
--- a/libglusterfs/src/glfs-message-id.h
+++ b/libglusterfs/src/glfs-message-id.h
@@ -162,6 +162,10 @@ GLFS_MSGID_COMP_SYMLINK_CACHE_END
 #define GLFS_MSGID_COMP_SHARD               GLFS_MSGID_COMP_CHANGELOG_LIB_END
 #define GLFS_MSGID_COMP_SHARD_END           (GLFS_MSGID_COMP_SHARD +\
                                              GLFS_MSGID_SEGMENT)
+
+#define GLFS_MSGID_COMP_NSR                     GLFS_MSGID_COMP_SHARD_END
+#define GLFS_MSGID_COMP_NSR_END                 (GLFS_MSGID_COMP_SHARD_END+\
+                                                 GLFS_MSGID_SEGMENT)
 /* --- new segments for messages goes above this line --- */
 
 #endif /* !_GLFS_MESSAGE_ID_H_ */
diff --git a/xlators/experimental/Makefile.am b/xlators/experimental/Makefile.am
index e182a87eb77..f937146c72a 100644
--- a/xlators/experimental/Makefile.am
+++ b/xlators/experimental/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS =
+SUBDIRS = nsr-client
 
 CLEANFILES =
diff --git a/xlators/experimental/nsr-client/Makefile.am b/xlators/experimental/nsr-client/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/experimental/nsr-client/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/experimental/nsr-client/src/Makefile.am b/xlators/experimental/nsr-client/src/Makefile.am
new file mode 100644
index 00000000000..0dbba285cc9
--- /dev/null
+++ b/xlators/experimental/nsr-client/src/Makefile.am
@@ -0,0 +1,32 @@
+xlator_LTLIBRARIES = nsrc.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
+
+nodist_nsrc_la_SOURCES = nsrc-cg.c
+CLEANFILES = $(nodist_nsrc_la_SOURCES)
+
+nsrc_la_LDFLAGS = -module -avoid-version
+nsrc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = \
+	 $(top_srcdir)/xlators/lib/src/libxlator.h \
+	 $(top_srcdir)/glusterfsd/src/glusterfsd.h \
+	 nsrc.h nsr-messages.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+NSRC_PREFIX	= $(top_srcdir)/xlators/experimental/nsr-client/src
+NSRC_GEN_FOPS	= $(NSRC_PREFIX)/gen-fops.py
+NSRC_TEMPLATES	= $(NSRC_PREFIX)/fop-template.c
+NSRC_WRAPPER	= $(NSRC_PREFIX)/nsrc.c
+noinst_PYTHON	= $(NSRC_GEN_FOPS)
+EXTRA_DIST	= $(NSRC_TEMPLATES) $(NSRC_WRAPPER)
+
+nsrc-cg.c: $(NSRC_GEN_FOPS) $(NSRC_TEMPLATES) $(NSRC_WRAPPER)
+	$(PYTHON) $(NSRC_GEN_FOPS) $(NSRC_TEMPLATES) $(NSRC_WRAPPER) > $@
+
+uninstall-local:
+	rm -f $(DESTDIR)$(xlatordir)/nsr.so
diff --git a/xlators/experimental/nsr-client/src/fop-template.c b/xlators/experimental/nsr-client/src/fop-template.c
new file mode 100644
index 00000000000..59708732aa0
--- /dev/null
+++ b/xlators/experimental/nsr-client/src/fop-template.c
@@ -0,0 +1,113 @@
+/* template-name fop */
+int32_t
+nsrc_@NAME@ (call_frame_t *frame, xlator_t *this,
+             @LONG_ARGS@)
+{
+        nsrc_local_t    *local          = NULL;
+        xlator_t        *target_xl      = ACTIVE_CHILD(this);
+
+        local = mem_get(this->local_pool);
+        if (!local) {
+                goto err;
+        }
+
+        local->stub = fop_@NAME@_stub (frame, nsrc_@NAME@_continue,
+                                       @SHORT_ARGS@);
+        if (!local->stub) {
+                goto err;
+        }
+        local->curr_xl = target_xl;
+        local->scars = 0;
+
+        frame->local = local;
+        STACK_WIND_COOKIE (frame, nsrc_@NAME@_cbk, target_xl,
+                    target_xl, target_xl->fops->@NAME@,
+                    @SHORT_ARGS@);
+        return 0;
+
+err:
+        if (local) {
+                mem_put(local);
+        }
+        STACK_UNWIND_STRICT (@NAME@, frame, -1, ENOMEM,
+                             @ERROR_ARGS@);
+        return 0;
+}
+
+/* template-name cbk */
+int32_t
+nsrc_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 @LONG_ARGS@)
+{
+        nsrc_local_t    *local          = frame->local;
+        xlator_t        *last_xl        = cookie;
+        xlator_t        *next_xl;
+        nsrc_private_t  *priv           = this->private;
+        struct timespec spec;
+
+        if (op_ret != (-1)) {
+                if (local->scars) {
+                        gf_msg (this->name, GF_LOG_INFO, 0, N_MSG_RETRY_MSG,
+                                HILITE("retried %p OK"), frame->local);
+                }
+                priv->active = last_xl;
+                goto unwind;
+        }
+        if ((op_errno != EREMOTE) && (op_errno != ENOTCONN)) {
+                goto unwind;
+        }
+
+        /* TBD: get leader ID from xdata? */
+        next_xl = next_xlator(this, last_xl);
+        /*
+         * We can't just give up after we've tried all bricks, because it's
+         * quite likely that a new leader election just hasn't finished yet.
+         * We also shouldn't retry endlessly, and especially not at a high
+         * rate, but that's good enough while we work on other things.
+         *
+         * TBD: implement slow/finite retry via a worker thread
+         */
+        if (!next_xl || (local->scars >= SCAR_LIMIT)) {
+                gf_msg (this->name, GF_LOG_DEBUG, 0, N_MSG_RETRY_MSG,
+                        HILITE("ran out of retries for %p"), frame->local);
+                goto unwind;
+        }
+
+        local->curr_xl = next_xl;
+        local->scars += 1;
+        spec.tv_sec = 1;
+        spec.tv_nsec = 0;
+        /*
+         * WARNING
+         *
+         * Just calling gf_timer_call_after like this leaves open the
+         * possibility that writes will get reordered, if a first write is
+         * rescheduled and then a second comes along to find an updated
+         * priv->active before the first actually executes.  We might need to
+         * implement a stricter (and more complicated) queuing mechanism to
+         * ensure absolute consistency in this case.
+         */
+        if (gf_timer_call_after(this->ctx, spec, nsrc_retry_cb, local)) {
+                return 0;
+        }
+
+unwind:
+        call_stub_destroy(local->stub);
+        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
+                                     @SHORT_ARGS@);
+        return 0;
+}
+
+/* template-name cont-func */
+int32_t
+nsrc_@NAME@_continue (call_frame_t *frame, xlator_t *this,
+                      @LONG_ARGS@)
+{
+        nsrc_local_t    *local  = frame->local;
+
+        STACK_WIND_COOKIE (frame, nsrc_@NAME@_cbk, local->curr_xl,
+                           local->curr_xl, local->curr_xl->fops->@NAME@,
+                           @SHORT_ARGS@);
+        return 0;
+}
diff --git a/xlators/experimental/nsr-client/src/gen-fops.py b/xlators/experimental/nsr-client/src/gen-fops.py
new file mode 100755
index 00000000000..4d9451f7177
--- /dev/null
+++ b/xlators/experimental/nsr-client/src/gen-fops.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+
+import os
+import re
+import string
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir,'../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, cbk_subs, generate
+
+# We really want the callback argument list, even when we're generating fop
+# code, so we propagate here.
+# TBD: this should probably be right in generate.py
+for k, v in cbk_subs.iteritems():
+	fop_subs[k]['@ERROR_ARGS@'] = v['@ERROR_ARGS@']
+
+# Stolen from old codegen.py
+def load_templates (path):
+	templates = {}
+	tmpl_re = re.compile("/\* template-name (.*) \*/")
+	templates = {}
+	t_name = None
+	for line in open(path,"r").readlines():
+		if not line:
+			break
+		m = tmpl_re.match(line)
+		if m:
+			if t_name:
+				templates[t_name] = string.join(t_contents,'')
+			t_name = m.group(1).strip()
+			t_contents = []
+		elif t_name:
+			t_contents.append(line)
+	if t_name:
+		templates[t_name] = string.join(t_contents,'')
+	return templates
+
+# Stolen from gen_fdl.py
+def gen_client (templates):
+	for name, value in ops.iteritems():
+		if name == 'getspec':
+			# It's not real if it doesn't have a stub function.
+			continue
+		print generate(templates['cbk'],name,cbk_subs)
+		print generate(templates['cont-func'],name,fop_subs)
+		print generate(templates['fop'],name,fop_subs)
+
+tmpl = load_templates(sys.argv[1])
+for l in open(sys.argv[2],'r').readlines():
+	if l.find('#pragma generate') != -1:
+		print "/* BEGIN GENERATED CODE - DO NOT MODIFY */"
+		gen_client(tmpl)
+		print "/* END GENERATED CODE */"
+	else:
+		print l[:-1]
diff --git a/xlators/experimental/nsr-client/src/nsr-messages.h b/xlators/experimental/nsr-client/src/nsr-messages.h
new file mode 100644
index 00000000000..d68cfc5d228
--- /dev/null
+++ b/xlators/experimental/nsr-client/src/nsr-messages.h
@@ -0,0 +1,67 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _NSR_MESSAGES_H_
+#define _NSR_MESSAGES_H_
+
+#include "glfs-message-id.h"
+
+/* NOTE: Rules for message additions
+ * 1) Each instance of a message is _better_ left with a unique message ID, even
+ *    if the message format is the same. Reasoning is that, if the message
+ *    format needs to change in one instance, the other instances are not
+ *    impacted or the new change does not change the ID of the instance being
+ *    modified.
+ * 2) Addition of a message,
+ *       - Should increment the GLFS_NUM_MESSAGES
+ *       - Append to the list of messages defined, towards the end
+ *       - Retain macro naming as glfs_msg_X (for redability across developers)
+ * NOTE: Rules for message format modifications
+ * 3) Check acorss the code if the message ID macro in question is reused
+ *    anywhere. If reused then then the modifications should ensure correctness
+ *    everywhere, or needs a new message ID as (1) above was not adhered to. If
+ *    not used anywhere, proceed with the required modification.
+ * NOTE: Rules for message deletion
+ * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
+ *    anywhere, then can be deleted, but will leave a hole by design, as
+ *    addition rules specify modification to the end of the list and not filling
+ *    holes.
+ */
+
+#define NSR_COMP_BASE           GLFS_MSGID_COMP_NSR
+#define GLFS_NUM_MESSAGES       1
+#define GLFS_MSGID_END          (NSR_COMP_BASE + GLFS_NUM_MESSAGES + 1)
+
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ */
+#define N_MSG_INIT_FAIL                      (NSR_COMP_BASE + 1)
+
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ */
+#define N_MSG_RETRY_MSG                      (NSR_COMP_BASE + 2)
+
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ */
+#define N_MSG_MEM_ERR                        (NSR_COMP_BASE + 3)
+
+
+
+
+
+#endif /* _NSR_MESSAGES_H_ */
diff --git a/xlators/experimental/nsr-client/src/nsrc.c b/xlators/experimental/nsr-client/src/nsrc.c
new file mode 100644
index 00000000000..dd3ad20544e
--- /dev/null
+++ b/xlators/experimental/nsr-client/src/nsrc.c
@@ -0,0 +1,236 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "call-stub.h"
+#include "defaults.h"
+#include "timer.h"
+#include "xlator.h"
+#include "nsr-messages.h"
+#include "nsrc.h"
+
+#define SCAR_LIMIT      20
+#define HILITE(x)       (""x"")
+
+/*
+ * The fops are actually generated by gen-fops.py; the rest was mostly copied
+ * from defaults.c (commit cd253754 on 27 August 2013).
+ */
+
+enum gf_dht_mem_types_ {
+        gf_mt_nsrc_private_t = gf_common_mt_end + 1,
+        gf_mt_nsrc_end
+};
+
+char            *NSRC_XATTR     = "user.nsr.active";
+
+static inline
+xlator_t *
+ACTIVE_CHILD (xlator_t *parent)
+{
+        nsrc_private_t  *priv   = parent->private;
+
+        return priv ? priv->active : FIRST_CHILD(parent);
+}
+
+xlator_t *
+next_xlator (xlator_t *this, xlator_t *prev)
+{
+        xlator_list_t   *trav;
+
+        for (trav = this->children; trav; trav = trav->next) {
+                if (trav->xlator == prev) {
+                        return trav->next ? trav->next->xlator
+                                          : this->children->xlator;
+                }
+        }
+
+        return NULL;
+}
+
+void
+nsrc_retry_cb (void *cb_arg)
+{
+        nsrc_local_t    *local  = cb_arg;
+
+        gf_msg (__func__, GF_LOG_INFO, 0, N_MSG_RETRY_MSG,
+                HILITE("retrying %p"), local);
+        call_resume_wind(local->stub);
+}
+
+#pragma generate
+
+int32_t
+nsrc_forget (xlator_t *this, inode_t *inode)
+{
+        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, N_MSG_INIT_FAIL,
+                          "xlator does not implement forget_cbk");
+        return 0;
+}
+
+
+int32_t
+nsrc_releasedir (xlator_t *this, fd_t *fd)
+{
+        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, N_MSG_INIT_FAIL,
+                          "xlator does not implement releasedir_cbk");
+        return 0;
+}
+
+int32_t
+nsrc_release (xlator_t *this, fd_t *fd)
+{
+        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, N_MSG_INIT_FAIL,
+                          "xlator does not implement release_cbk");
+        return 0;
+}
+
+struct xlator_fops fops = {
+        .lookup         = nsrc_lookup,
+        .stat           = nsrc_stat,
+        .fstat          = nsrc_fstat,
+        .truncate       = nsrc_truncate,
+        .ftruncate      = nsrc_ftruncate,
+        .access         = nsrc_access,
+        .readlink       = nsrc_readlink,
+        .mknod          = nsrc_mknod,
+        .mkdir          = nsrc_mkdir,
+        .unlink         = nsrc_unlink,
+        .rmdir          = nsrc_rmdir,
+        .symlink        = nsrc_symlink,
+        .rename         = nsrc_rename,
+        .link           = nsrc_link,
+        .create         = nsrc_create,
+        .open           = nsrc_open,
+        .readv          = nsrc_readv,
+        .writev         = nsrc_writev,
+        .flush          = nsrc_flush,
+        .fsync          = nsrc_fsync,
+        .opendir        = nsrc_opendir,
+        .readdir        = nsrc_readdir,
+        .readdirp       = nsrc_readdirp,
+        .fsyncdir       = nsrc_fsyncdir,
+        .statfs         = nsrc_statfs,
+        .setxattr       = nsrc_setxattr,
+        .getxattr       = nsrc_getxattr,
+        .fsetxattr      = nsrc_fsetxattr,
+        .fgetxattr      = nsrc_fgetxattr,
+        .removexattr    = nsrc_removexattr,
+        .fremovexattr   = nsrc_fremovexattr,
+        .lk             = nsrc_lk,
+        .inodelk        = nsrc_inodelk,
+        .finodelk       = nsrc_finodelk,
+        .entrylk        = nsrc_entrylk,
+        .fentrylk       = nsrc_fentrylk,
+        .rchecksum      = nsrc_rchecksum,
+        .xattrop        = nsrc_xattrop,
+        .fxattrop       = nsrc_fxattrop,
+        .setattr        = nsrc_setattr,
+        .fsetattr       = nsrc_fsetattr,
+	.fallocate	= nsrc_fallocate,
+	.discard        = nsrc_discard,
+};
+
+struct xlator_cbks cbks = {
+};
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("nsrc", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_mt_nsrc_end + 1);
+
+        if (ret != 0) {
+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, N_MSG_MEM_ERR,
+                        "Memory accounting init failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+int32_t
+nsrc_init (xlator_t *this)
+{
+        nsrc_private_t  *priv   = NULL;
+
+        this->local_pool = mem_pool_new (nsrc_local_t, 128);
+        if (!this->local_pool) {
+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, N_MSG_MEM_ERR,
+                        "failed to create nsrc_local_t pool");
+                goto err;
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_mt_nsrc_private_t);
+        if (!priv) {
+                goto err;
+        }
+
+        priv->active = FIRST_CHILD(this);
+        this->private = priv;
+        return 0;
+
+err:
+        if (priv) {
+                GF_FREE(priv);
+        }
+        return -1;
+}
+
+void
+nsrc_fini (xlator_t *this)
+{
+        GF_FREE(this->private);
+}
+
+int32_t
+nsrc_notify (xlator_t *this, int32_t event, void *data, ...)
+{
+        int32_t         ret     = 0;
+
+        switch (event) {
+        case GF_EVENT_CHILD_DOWN:
+                /*
+                 * TBD: handle this properly
+                 *
+                 * What we really should do is propagate this only if it caused
+                 * us to lose quorum, and likewise for GF_EVENT_CHILD_UP only
+                 * if it caused us to gain quorum.  However, that requires
+                 * tracking child states and for now it's easier to swallow
+                 * these unconditionally.  The consequence of failing to do
+                 * this is that DHT sees the first GF_EVENT_CHILD_DOWN and gets
+                 * confused, so it doesn't call us and doesn't get up-to-date
+                 * directory listings etc.
+                 */
+                break;
+        default:
+                ret = default_notify (this, event, data);
+        }
+
+        return ret;
+}
+
+class_methods_t class_methods = {
+        .init           = nsrc_init,
+        .fini           = nsrc_fini,
+        .notify         = nsrc_notify,
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/experimental/nsr-client/src/nsrc.h b/xlators/experimental/nsr-client/src/nsrc.h
new file mode 100644
index 00000000000..0c61d7a9fa8
--- /dev/null
+++ b/xlators/experimental/nsr-client/src/nsrc.h
@@ -0,0 +1,24 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _NSRC_H_
+#define _NSRC_H_
+
+typedef struct {
+        xlator_t        *active;
+} nsrc_private_t;
+
+typedef struct {
+        call_stub_t     *stub;
+        xlator_t        *curr_xl;
+        uint16_t        scars;
+} nsrc_local_t;
+
+#endif /* _NSRC_H_ */